From afa8b6ca2485df3147c5146b564ab10fcbf5a266 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 20 Nov 2025 11:52:50 +0800 Subject: [PATCH 01/30] Recognizes expectedDataGeneration = -1L as a sentinel for "create if not exists" (If-None-Match) semantics. --- .../hadoop/ozone/client/OzoneBucket.java | 4 +-- .../hadoop/ozone/client/rpc/RpcClient.java | 4 +-- .../ozone/om/exceptions/OMException.java | 6 +++-- .../om/request/key/OMKeyCommitRequest.java | 26 +++++++++++++------ .../om/request/key/OMKeyCreateRequest.java | 25 ++++++++++++------ 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index b9649f09bb15..64a9a9317053 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -1029,8 +1029,8 @@ public List listStatus(String keyName, boolean recursive, * * @param prefix Optional string to filter for the selected keys. */ - public OzoneMultipartUploadList listMultipartUploads(String prefix, - String keyMarker, String uploadIdMarker, int maxUploads) + public OzoneMultipartUploadList listMultipartUploads(String prefix, + String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { return proxy.listMultipartUploads(volumeName, getName(), prefix, keyMarker, uploadIdMarker, maxUploads); } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 4f6ddd76bafd..1c762f7a0367 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -683,7 +683,7 @@ public void createBucket( builder.setDefaultReplicationConfig(defaultReplicationConfig); } - String replicationType = defaultReplicationConfig == null + String replicationType = defaultReplicationConfig == null ? "server-side default replication type" : defaultReplicationConfig.getType().toString(); @@ -1321,7 +1321,7 @@ public List listBuckets(String volumeName, String bucketPrefix, List buckets = ozoneManagerClient.listBuckets( volumeName, prevBucket, bucketPrefix, maxListResult, hasSnapshot); - return buckets.stream().map(bucket -> + return buckets.stream().map(bucket -> OzoneBucket.newBuilder(conf, this) .setVolumeName(bucket.getVolumeName()) .setName(bucket.getBucketName()) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 596eb1276560..7f47c56fbf2f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -128,6 +128,8 @@ public enum ResultCodes { KEY_NOT_FOUND, + KEY_GENERATION_MISMATCH, + INVALID_KEY_NAME, ACCESS_DENIED, @@ -208,7 +210,7 @@ public enum ResultCodes { USER_MISMATCH, // Error code when requested user name passed is different // from remote user. - INVALID_PART, // When part name is not found or not matching with partname + INVALID_PART, // When part name is not found or not matching with partname // in OM MPU partInfo. INVALID_PART_ORDER, // When list of parts mentioned to complete MPU are not @@ -267,7 +269,7 @@ public enum ResultCodes { UNAUTHORIZED, S3_SECRET_ALREADY_EXISTS, - + INVALID_PATH, TOO_MANY_BUCKETS, KEY_UNDER_LEASE_RECOVERY, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index a106903bce79..6fae1b2b7721 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.request.key; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_CLOSED; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_UNDER_LEASE_RECOVERY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; @@ -616,14 +617,23 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map if (toCommit.getExpectedDataGeneration() != null) { // These values are not passed in the request keyArgs, so add them into the auditMap if they are present // in the open key entry. - auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(toCommit.getExpectedDataGeneration())); - if (existing == null) { - throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); - } - if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { - throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + - ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", - KEY_NOT_FOUND); + Long expectedGen = toCommit.getExpectedDataGeneration(); + auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(expectedGen)); + + if (expectedGen == -1L) { + if (existing != null) { + throw new OMException("Key already exists", + OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } else { + if (existing == null) { + throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); + } + if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { + throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + + ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", + KEY_GENERATION_MISMATCH); + } } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 6040cb7ddf6d..c33575cd819a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -178,7 +178,7 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { KeyArgs.Builder finalNewKeyArgs = newKeyArgs; KeyArgs resolvedKeyArgs = - captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), + captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), () -> resolveBucketAndCheckKeyAcls(finalNewKeyArgs.build(), ozoneManager, IAccessAuthorizer.ACLType.CREATE)); newCreateKeyRequest = @@ -358,7 +358,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } else { perfMetrics.addCreateKeyFailureLatencyNs(createKeyLatency); } - + if (acquireLock) { mergeOmLockDetails(ozoneLockStrategy .releaseWriteLock(omMetadataManager, volumeName, @@ -460,12 +460,21 @@ public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throws OMException { if (keyArgs.hasExpectedDataGeneration()) { - // If a key does not exist, or if it exists but the updateID do not match, then fail this request. - if (dbKeyInfo == null) { - throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); - } - if (dbKeyInfo.getUpdateID() != keyArgs.getExpectedDataGeneration()) { - throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + long expectedGen = keyArgs.getExpectedDataGeneration(); + // If expectedGen is -1, it means the key MUST NOT exist (If-None-Match) + if (expectedGen == -1L) { + if (dbKeyInfo != null) { + throw new OMException("Key already exists", + OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } else { + // If a key does not exist, or if it exists but the updateID do not match, then fail this request. + if (dbKeyInfo == null) { + throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + } + if (dbKeyInfo.getUpdateID() != expectedGen) { + throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + } } } } From 7868a862ff616154e4bd9ab52dada929e15042ec Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 25 Nov 2025 00:32:30 +0800 Subject: [PATCH 02/30] correct the new result code position --- .../org/apache/hadoop/ozone/om/exceptions/OMException.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 7f47c56fbf2f..c95cfc162ba2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -128,8 +128,6 @@ public enum ResultCodes { KEY_NOT_FOUND, - KEY_GENERATION_MISMATCH, - INVALID_KEY_NAME, ACCESS_DENIED, @@ -277,5 +275,7 @@ public enum ResultCodes { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD, TOO_MANY_SNAPSHOTS, + + KEY_GENERATION_MISMATCH, } } From da492ce17eec5721c5dd1c13519bf61a62150879 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 3 Dec 2025 23:38:50 +0800 Subject: [PATCH 03/30] HDDS-14070. Update key generation mismatch handling in Ozone RPC client tests and protocol definition. --- .../apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java | 3 ++- .../interface-client/src/main/proto/OmClientProtocol.proto | 2 ++ .../hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 554b125ddbec..646ea2b618fd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -39,6 +39,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.MD5_HASH; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; @@ -1356,7 +1357,7 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE keyInfo = ozoneManager.lookupKey(keyArgs); OMException e = assertThrows(OMException.class, out::close); - assertEquals(KEY_NOT_FOUND, e.getResult()); + assertEquals(KEY_GENERATION_MISMATCH, e.getResult()); assertThat(e).hasMessageContaining("does not match the expected generation to rewrite"); } finally { if (out != null) { diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index bdb3cc3cee35..06cbf60ea746 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -566,6 +566,8 @@ enum Status { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97; TOO_MANY_SNAPSHOTS = 98; + + KEY_GENERATION_MISMATCH = 99; } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index e8bd2b079416..dc09fcae2623 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om.request.key; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.assertj.core.api.Assertions.assertThat; @@ -262,7 +263,7 @@ public void testAtomicRewrite() throws Exception { closedKeyTable.put(getOzonePathKey(), invalidKeyInfo); // This should fail as the updateID ia zero and the open key has rewrite generation of 1. omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(KEY_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); + assertEquals(KEY_GENERATION_MISMATCH, omClientResponse.getOMResponse().getStatus()); omKeyInfoBuilder.setUpdateID(1L); OmKeyInfo closedKeyInfo = omKeyInfoBuilder.build(); @@ -458,7 +459,7 @@ private Map doKeyCommit(boolean isHSync, .collect(Collectors.toList()); String openKey = addKeyToOpenKeyTable(allocatedBlockList); String ozoneKey = getOzonePathKey(); - + OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); assertEquals(OK, From 0d4dfb0a35e730ea172323f52cc9648ecba7dafa Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 4 Dec 2025 00:05:28 +0800 Subject: [PATCH 04/30] Introduce EXPECTED_GEN_CREATE_IF_NOT_EXISTS constant --- .../src/main/java/org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyCreateRequest.java | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 42ca3f97b3b0..3d588e7ec006 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -314,6 +314,7 @@ public final class OzoneConsts { public static final String TENANT = "tenant"; public static final String USER_PREFIX = "userPrefix"; public static final String REWRITE_GENERATION = "rewriteGeneration"; + public static final long EXPECTED_GEN_CREATE_IF_NOT_EXISTS = -1L; public static final String FROM_SNAPSHOT = "fromSnapshot"; public static final String TO_SNAPSHOT = "toSnapshot"; public static final String TOKEN = "token"; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index e4eff3366c70..7bf705c9f64b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -620,7 +620,7 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map Long expectedGen = toCommit.getExpectedDataGeneration(); auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(expectedGen)); - if (expectedGen == -1L) { + if (expectedGen == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { if (existing != null) { throw new OMException("Key already exists", OMException.ResultCodes.KEY_ALREADY_EXISTS); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index c33575cd819a..febdba85f46c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.utils.UniqueId; import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -461,8 +462,8 @@ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throws OMException { if (keyArgs.hasExpectedDataGeneration()) { long expectedGen = keyArgs.getExpectedDataGeneration(); - // If expectedGen is -1, it means the key MUST NOT exist (If-None-Match) - if (expectedGen == -1L) { + // If expectedGen is EXPECTED_GEN_CREATE_IF_NOT_EXISTS, it means the key MUST NOT exist (If-None-Match) + if (expectedGen == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { if (dbKeyInfo != null) { throw new OMException("Key already exists", OMException.ResultCodes.KEY_ALREADY_EXISTS); From 4e9665c29bf31ccb74b2330279a7de8e7475b0b8 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 2 Jan 2026 15:50:56 +0800 Subject: [PATCH 05/30] revert the new KEY_GENERATION_MISMATCH error code --- .../org/apache/hadoop/ozone/om/exceptions/OMException.java | 2 -- .../apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java | 2 +- .../interface-client/src/main/proto/OmClientProtocol.proto | 2 -- .../hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 3 +-- .../hadoop/ozone/om/request/key/OMKeyCreateRequest.java | 3 ++- .../hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java | 4 ++-- 6 files changed, 6 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index c95cfc162ba2..2b5b5559f92b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -275,7 +275,5 @@ public enum ResultCodes { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD, TOO_MANY_SNAPSHOTS, - - KEY_GENERATION_MISMATCH, } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 1a6cb4a9b36c..6ab484e0fe61 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -40,7 +40,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 06cbf60ea746..bdb3cc3cee35 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -566,8 +566,6 @@ enum Status { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97; TOO_MANY_SNAPSHOTS = 98; - - KEY_GENERATION_MISMATCH = 99; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index cb2aacbdd63b..4d2ce035e9d9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om.request.key; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_CLOSED; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_UNDER_LEASE_RECOVERY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; @@ -632,7 +631,7 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", - KEY_GENERATION_MISMATCH); + KEY_NOT_FOUND); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index cd4f502e2fd8..2a17df2c7fd2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -474,7 +474,8 @@ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); } if (dbKeyInfo.getUpdateID() != expectedGen) { - throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + throw new OMException("Generation mismatch during expected rewrite", + OMException.ResultCodes.KEY_NOT_FOUND); } } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index cf099d9c2b45..3d0eedec2fb4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.om.request.key; -import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_GENERATION_MISMATCH; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.assertj.core.api.Assertions.assertThat; @@ -261,7 +261,7 @@ public void testAtomicRewrite() throws Exception { closedKeyTable.put(getOzonePathKey(), invalidKeyInfo); // This should fail as the updateID ia zero and the open key has rewrite generation of 1. omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(KEY_GENERATION_MISMATCH, omClientResponse.getOMResponse().getStatus()); + assertEquals(KEY_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); omKeyInfoBuilder.setUpdateID(1L); OmKeyInfo closedKeyInfo = omKeyInfoBuilder.build(); From b5249f1d8a1499a319f5bc8daa20b608f17d64b0 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 2 Jan 2026 15:59:37 +0800 Subject: [PATCH 06/30] Add tests for atomic key creation and rewriting with expected generation handling - Implemented tests for `createKey` and `rewriteKey` methods to validate behavior when using the `EXPECTED_GEN_CREATE_IF_NOT_EXISTS` constant. - Added scenarios for key creation when the key is absent and when it already exists. - Enhanced the `rewriteFailsWhenKeyExists` test to cover cases for both committed and uncommitted keys. - Updated error handling to ensure correct responses for key existence checks. --- .../ozone/client/rpc/OzoneRpcClientTests.java | 57 ++++++++++++- .../request/key/TestOMKeyCommitRequest.java | 70 ++++++++++++++++ .../request/key/TestOMKeyCreateRequest.java | 84 +++++++++++++++++++ 3 files changed, 210 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 6ab484e0fe61..879db6f7edca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -35,6 +35,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.DEFAULT_OM_UPDATE_ID; import static org.apache.hadoop.ozone.OzoneConsts.ETAG; +import static org.apache.hadoop.ozone.OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS; import static org.apache.hadoop.ozone.OzoneConsts.GB; import static org.apache.hadoop.ozone.OzoneConsts.MD5_HASH; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; @@ -1358,7 +1359,7 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE keyInfo = ozoneManager.lookupKey(keyArgs); OMException e = assertThrows(OMException.class, out::close); - assertEquals(KEY_GENERATION_MISMATCH, e.getResult()); + assertEquals(KEY_NOT_FOUND, e.getResult()); assertThat(e).hasMessageContaining("does not match the expected generation to rewrite"); } finally { if (out != null) { @@ -1371,6 +1372,54 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE assertUnchanged(keyInfo, ozoneManager.lookupKey(keyArgs)); } + @ParameterizedTest + @EnumSource + void rewriteFailsWhenKeyExists(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails key1Details = createTestKey(bucket, "key1", "value".getBytes(UTF_8)); + OzoneOutputStream key2Out = openTestKey(bucket, "key2", "value"); + OzoneOutputStream key3Out = openTestKey(bucket, "key3", "value"); + + // Test 1: Rewrite with -1 fails when key is already committed + OMException e = assertThrows(OMException.class, () -> { + bucket.rewriteKey( + key1Details.getName(), + key1Details.getDataSize(), + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + key1Details.getMetadata()); + }); + + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + assertThat(e).hasMessageContaining("Key already exists"); + + // Test 2: Rewrite with -1 succeeds when key is open but not yet committed + assertDoesNotThrow(() -> { + bucket.rewriteKey("key2", + 1024, + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", "value")); + }); + key2Out.close(); + + // Test 3: After rewrite completes, attempting to rewrite again with -1 fails + key3Out.write("value".getBytes(UTF_8)); + key3Out.close(); + + e = assertThrows(OMException.class, () -> { + bucket.rewriteKey("key2", + 1024, + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", "value")); + }); + + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + assertThat(e).hasMessageContaining("Key already exists"); + } + @ParameterizedTest @EnumSource void cannotRewriteDeletedKey(BucketLayout layout) throws IOException { @@ -4386,6 +4435,12 @@ private void completeMultipartUpload(OzoneBucket bucket, String keyName, assertNotNull(omMultipartUploadCompleteInfo.getHash()); } + private OzoneOutputStream openTestKey(OzoneBucket bucket, String keyName, String keyValue) throws IOException { + return bucket.createKey(keyName, keyValue.getBytes(UTF_8).length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", RandomStringUtils.secure().nextAscii(10))); + } + private OzoneKeyDetails createTestKey(OzoneBucket bucket) throws IOException { return createTestKey(bucket, getTestName(), UUID.randomUUID().toString()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 3d0eedec2fb4..e1ba052b821e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -280,6 +280,76 @@ public void testAtomicRewrite() throws Exception { assertEquals(acls, committedKey.getAcls()); } + @Test + public void testAtomicCreateIfNotExistsCommitKeyAbsent() throws Exception { + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedDataGeneration(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + assertNull(closedKeyTable.get(getOzonePathKey())); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); + + OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); + assertNotNull(committedKey); + assertNull(committedKey.getExpectedDataGeneration()); + } + + @Test + public void testAtomicCreateIfNotExistsCommitKeyAlreadyExists() throws Exception { + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedDataGeneration(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + OmKeyInfo existingClosedKey = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())).build(); + closedKeyTable.put(getOzonePathKey(), existingClosedKey); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_ALREADY_EXISTS, omClientResponse.getOMResponse().getStatus()); + } + @Test public void testValidateAndUpdateCacheWithUncommittedBlocks() throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index b4cf5f7cc142..ecdc21745f50 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -27,6 +27,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.addVolumeAndBucketToDB; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.createOmKeyInfo; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.NOT_A_FILE; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; @@ -134,6 +135,89 @@ public void preExecuteRejectsInvalidReplication() { assertEquals(OMException.ResultCodes.INVALID_REQUEST, e.getResult()); } + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenCreateIfNotExistsKeyMissing( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + + checkResponse(modifiedOmRequest, response, id, false, getBucketLayout()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenCreateIfNotExistsKeyAlreadyExists( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig).setUpdateID(1L).build(); + omMetadataManager.getKeyTable(getBucketLayout()).put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + String openKey = getOpenKey(id); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_ALREADY_EXISTS, response.getOMResponse().getStatus()); + + // As we got error, no entry should be created in openKeyTable. + OmKeyInfo openKeyInfo = + omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + assertNull(openKeyInfo); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenMismatchReturnsKeyGenerationMismatch( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + long expectedGen = 1L; + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, expectedGen)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig).setUpdateID(2L).build(); + omMetadataManager.getKeyTable(getBucketLayout()).put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + String openKey = getOpenKey(id); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus()); + + // As we got error, no entry should be created in openKeyTable. + OmKeyInfo openKeyInfo = + omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + assertNull(openKeyInfo); + } + @ParameterizedTest @MethodSource("data") public void testValidateAndUpdateCache( From 817e33fd9feca9f91f9672b9ea5b1eea851d461e Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 20 Nov 2025 11:52:50 +0800 Subject: [PATCH 07/30] Recognizes expectedDataGeneration = -1L as a sentinel for "create if not exists" (If-None-Match) semantics. --- .../hadoop/ozone/client/OzoneBucket.java | 4 +-- .../hadoop/ozone/client/rpc/RpcClient.java | 4 +-- .../ozone/om/exceptions/OMException.java | 6 +++-- .../om/request/key/OMKeyCommitRequest.java | 26 +++++++++++++------ .../om/request/key/OMKeyCreateRequest.java | 25 ++++++++++++------ 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 566427424223..4b9a3cd0f3d5 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -1029,8 +1029,8 @@ public List listStatus(String keyName, boolean recursive, * * @param prefix Optional string to filter for the selected keys. */ - public OzoneMultipartUploadList listMultipartUploads(String prefix, - String keyMarker, String uploadIdMarker, int maxUploads) + public OzoneMultipartUploadList listMultipartUploads(String prefix, + String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { return proxy.listMultipartUploads(volumeName, getName(), prefix, keyMarker, uploadIdMarker, maxUploads); } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 3947e4b6818b..7b1c1a37bf04 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -677,7 +677,7 @@ public void createBucket( builder.setDefaultReplicationConfig(defaultReplicationConfig); } - String replicationType = defaultReplicationConfig == null + String replicationType = defaultReplicationConfig == null ? "server-side default replication type" : defaultReplicationConfig.getType().toString(); @@ -1315,7 +1315,7 @@ public List listBuckets(String volumeName, String bucketPrefix, List buckets = ozoneManagerClient.listBuckets( volumeName, prevBucket, bucketPrefix, maxListResult, hasSnapshot); - return buckets.stream().map(bucket -> + return buckets.stream().map(bucket -> OzoneBucket.newBuilder(conf, this) .setVolumeName(bucket.getVolumeName()) .setName(bucket.getBucketName()) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 596eb1276560..7f47c56fbf2f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -128,6 +128,8 @@ public enum ResultCodes { KEY_NOT_FOUND, + KEY_GENERATION_MISMATCH, + INVALID_KEY_NAME, ACCESS_DENIED, @@ -208,7 +210,7 @@ public enum ResultCodes { USER_MISMATCH, // Error code when requested user name passed is different // from remote user. - INVALID_PART, // When part name is not found or not matching with partname + INVALID_PART, // When part name is not found or not matching with partname // in OM MPU partInfo. INVALID_PART_ORDER, // When list of parts mentioned to complete MPU are not @@ -267,7 +269,7 @@ public enum ResultCodes { UNAUTHORIZED, S3_SECRET_ALREADY_EXISTS, - + INVALID_PATH, TOO_MANY_BUCKETS, KEY_UNDER_LEASE_RECOVERY, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index be0935d909d8..e63cdebf4164 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.request.key; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_CLOSED; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_UNDER_LEASE_RECOVERY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; @@ -616,14 +617,23 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map if (toCommit.getExpectedDataGeneration() != null) { // These values are not passed in the request keyArgs, so add them into the auditMap if they are present // in the open key entry. - auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(toCommit.getExpectedDataGeneration())); - if (existing == null) { - throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); - } - if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { - throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + - ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", - KEY_NOT_FOUND); + Long expectedGen = toCommit.getExpectedDataGeneration(); + auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(expectedGen)); + + if (expectedGen == -1L) { + if (existing != null) { + throw new OMException("Key already exists", + OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } else { + if (existing == null) { + throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); + } + if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { + throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + + ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", + KEY_GENERATION_MISMATCH); + } } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index d34320ecb8db..9d1519e455f4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -189,7 +189,7 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { KeyArgs.Builder finalNewKeyArgs = newKeyArgs; KeyArgs resolvedKeyArgs = - captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), + captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), () -> resolveBucketAndCheckKeyAcls(finalNewKeyArgs.build(), ozoneManager, IAccessAuthorizer.ACLType.CREATE)); newCreateKeyRequest = @@ -369,7 +369,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } else { perfMetrics.addCreateKeyFailureLatencyNs(createKeyLatency); } - + if (acquireLock) { mergeOmLockDetails(ozoneLockStrategy .releaseWriteLock(omMetadataManager, volumeName, @@ -471,12 +471,21 @@ public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throws OMException { if (keyArgs.hasExpectedDataGeneration()) { - // If a key does not exist, or if it exists but the updateID do not match, then fail this request. - if (dbKeyInfo == null) { - throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); - } - if (dbKeyInfo.getUpdateID() != keyArgs.getExpectedDataGeneration()) { - throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + long expectedGen = keyArgs.getExpectedDataGeneration(); + // If expectedGen is -1, it means the key MUST NOT exist (If-None-Match) + if (expectedGen == -1L) { + if (dbKeyInfo != null) { + throw new OMException("Key already exists", + OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } else { + // If a key does not exist, or if it exists but the updateID do not match, then fail this request. + if (dbKeyInfo == null) { + throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + } + if (dbKeyInfo.getUpdateID() != expectedGen) { + throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + } } } } From 021747bd790e05b00f038b65e7ac483175617a87 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 25 Nov 2025 00:32:30 +0800 Subject: [PATCH 08/30] correct the new result code position --- .../org/apache/hadoop/ozone/om/exceptions/OMException.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 7f47c56fbf2f..c95cfc162ba2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -128,8 +128,6 @@ public enum ResultCodes { KEY_NOT_FOUND, - KEY_GENERATION_MISMATCH, - INVALID_KEY_NAME, ACCESS_DENIED, @@ -277,5 +275,7 @@ public enum ResultCodes { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD, TOO_MANY_SNAPSHOTS, + + KEY_GENERATION_MISMATCH, } } From 0fd6730cc444d835f6f1057dbaeb11ee2fa878b7 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 3 Dec 2025 23:38:50 +0800 Subject: [PATCH 09/30] HDDS-14070. Update key generation mismatch handling in Ozone RPC client tests and protocol definition. --- .../apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java | 3 ++- .../interface-client/src/main/proto/OmClientProtocol.proto | 2 ++ .../hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 31c826fabb04..4e2eec109cf7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -40,6 +40,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; @@ -1354,7 +1355,7 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE keyInfo = ozoneManager.lookupKey(keyArgs); OMException e = assertThrows(OMException.class, out::close); - assertEquals(KEY_NOT_FOUND, e.getResult()); + assertEquals(KEY_GENERATION_MISMATCH, e.getResult()); assertThat(e).hasMessageContaining("does not match the expected generation to rewrite"); } finally { if (out != null) { diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index bdb3cc3cee35..06cbf60ea746 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -566,6 +566,8 @@ enum Status { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97; TOO_MANY_SNAPSHOTS = 98; + + KEY_GENERATION_MISMATCH = 99; } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 02d913160bc1..0d9d5e82ccb2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om.request.key; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.assertj.core.api.Assertions.assertThat; @@ -260,7 +261,7 @@ public void testAtomicRewrite() throws Exception { closedKeyTable.put(getOzonePathKey(), invalidKeyInfo); // This should fail as the updateID ia zero and the open key has rewrite generation of 1. omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(KEY_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); + assertEquals(KEY_GENERATION_MISMATCH, omClientResponse.getOMResponse().getStatus()); omKeyInfoBuilder.setUpdateID(1L); OmKeyInfo closedKeyInfo = omKeyInfoBuilder.build(); @@ -456,7 +457,7 @@ private Map doKeyCommit(boolean isHSync, .collect(Collectors.toList()); String openKey = addKeyToOpenKeyTable(allocatedBlockList); String ozoneKey = getOzonePathKey(); - + OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); assertEquals(OK, From c64beca5bc35e34bdffbf3d18d70c4984fa59eb3 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 4 Dec 2025 00:05:28 +0800 Subject: [PATCH 10/30] Introduce EXPECTED_GEN_CREATE_IF_NOT_EXISTS constant --- .../src/main/java/org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyCreateRequest.java | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 3bd8388f9500..0e8ea9d677fb 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -318,6 +318,7 @@ public final class OzoneConsts { public static final String TENANT = "tenant"; public static final String USER_PREFIX = "userPrefix"; public static final String REWRITE_GENERATION = "rewriteGeneration"; + public static final long EXPECTED_GEN_CREATE_IF_NOT_EXISTS = -1L; public static final String FROM_SNAPSHOT = "fromSnapshot"; public static final String TO_SNAPSHOT = "toSnapshot"; public static final String TOKEN = "token"; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index e63cdebf4164..cb2aacbdd63b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -620,7 +620,7 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map Long expectedGen = toCommit.getExpectedDataGeneration(); auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(expectedGen)); - if (expectedGen == -1L) { + if (expectedGen == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { if (existing != null) { throw new OMException("Key already exists", OMException.ResultCodes.KEY_ALREADY_EXISTS); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 9d1519e455f4..cb26a5429563 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.utils.UniqueId; import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -472,8 +473,8 @@ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throws OMException { if (keyArgs.hasExpectedDataGeneration()) { long expectedGen = keyArgs.getExpectedDataGeneration(); - // If expectedGen is -1, it means the key MUST NOT exist (If-None-Match) - if (expectedGen == -1L) { + // If expectedGen is EXPECTED_GEN_CREATE_IF_NOT_EXISTS, it means the key MUST NOT exist (If-None-Match) + if (expectedGen == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { if (dbKeyInfo != null) { throw new OMException("Key already exists", OMException.ResultCodes.KEY_ALREADY_EXISTS); From 6b33766f61965938b25e0b5392c744edb899c95d Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 2 Jan 2026 15:50:56 +0800 Subject: [PATCH 11/30] revert the new KEY_GENERATION_MISMATCH error code --- .../org/apache/hadoop/ozone/om/exceptions/OMException.java | 2 -- .../apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java | 2 +- .../interface-client/src/main/proto/OmClientProtocol.proto | 2 -- .../hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 3 +-- .../hadoop/ozone/om/request/key/OMKeyCreateRequest.java | 3 ++- .../hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java | 4 ++-- 6 files changed, 6 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index c95cfc162ba2..2b5b5559f92b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -275,7 +275,5 @@ public enum ResultCodes { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD, TOO_MANY_SNAPSHOTS, - - KEY_GENERATION_MISMATCH, } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 4e2eec109cf7..60a02cf370e7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -40,7 +40,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 06cbf60ea746..bdb3cc3cee35 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -566,8 +566,6 @@ enum Status { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97; TOO_MANY_SNAPSHOTS = 98; - - KEY_GENERATION_MISMATCH = 99; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index cb2aacbdd63b..4d2ce035e9d9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om.request.key; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_CLOSED; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_GENERATION_MISMATCH; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_UNDER_LEASE_RECOVERY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; @@ -632,7 +631,7 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", - KEY_GENERATION_MISMATCH); + KEY_NOT_FOUND); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index cb26a5429563..2581195adde4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -485,7 +485,8 @@ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); } if (dbKeyInfo.getUpdateID() != expectedGen) { - throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + throw new OMException("Generation mismatch during expected rewrite", + OMException.ResultCodes.KEY_NOT_FOUND); } } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 0d9d5e82ccb2..905b0f9b3a32 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.om.request.key; -import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_GENERATION_MISMATCH; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.assertj.core.api.Assertions.assertThat; @@ -261,7 +261,7 @@ public void testAtomicRewrite() throws Exception { closedKeyTable.put(getOzonePathKey(), invalidKeyInfo); // This should fail as the updateID ia zero and the open key has rewrite generation of 1. omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(KEY_GENERATION_MISMATCH, omClientResponse.getOMResponse().getStatus()); + assertEquals(KEY_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); omKeyInfoBuilder.setUpdateID(1L); OmKeyInfo closedKeyInfo = omKeyInfoBuilder.build(); From 83ef2ea8c20ebb0c3b1e1d3832d6baf66fdc7297 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 2 Jan 2026 15:59:37 +0800 Subject: [PATCH 12/30] Add tests for atomic key creation and rewriting with expected generation handling - Implemented tests for `createKey` and `rewriteKey` methods to validate behavior when using the `EXPECTED_GEN_CREATE_IF_NOT_EXISTS` constant. - Added scenarios for key creation when the key is absent and when it already exists. - Enhanced the `rewriteFailsWhenKeyExists` test to cover cases for both committed and uncommitted keys. - Updated error handling to ensure correct responses for key existence checks. --- .../ozone/client/rpc/OzoneRpcClientTests.java | 57 ++++++++++++- .../request/key/TestOMKeyCommitRequest.java | 70 ++++++++++++++++ .../request/key/TestOMKeyCreateRequest.java | 84 +++++++++++++++++++ 3 files changed, 210 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 60a02cf370e7..11a831bd7f0f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -35,6 +35,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.DEFAULT_OM_UPDATE_ID; import static org.apache.hadoop.ozone.OzoneConsts.ETAG; +import static org.apache.hadoop.ozone.OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS; import static org.apache.hadoop.ozone.OzoneConsts.GB; import static org.apache.hadoop.ozone.OzoneConsts.MD5_HASH; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; @@ -1355,7 +1356,7 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE keyInfo = ozoneManager.lookupKey(keyArgs); OMException e = assertThrows(OMException.class, out::close); - assertEquals(KEY_GENERATION_MISMATCH, e.getResult()); + assertEquals(KEY_NOT_FOUND, e.getResult()); assertThat(e).hasMessageContaining("does not match the expected generation to rewrite"); } finally { if (out != null) { @@ -1368,6 +1369,54 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE assertUnchanged(keyInfo, ozoneManager.lookupKey(keyArgs)); } + @ParameterizedTest + @EnumSource + void rewriteFailsWhenKeyExists(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails key1Details = createTestKey(bucket, "key1", "value".getBytes(UTF_8)); + OzoneOutputStream key2Out = openTestKey(bucket, "key2", "value"); + OzoneOutputStream key3Out = openTestKey(bucket, "key3", "value"); + + // Test 1: Rewrite with -1 fails when key is already committed + OMException e = assertThrows(OMException.class, () -> { + bucket.rewriteKey( + key1Details.getName(), + key1Details.getDataSize(), + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + key1Details.getMetadata()); + }); + + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + assertThat(e).hasMessageContaining("Key already exists"); + + // Test 2: Rewrite with -1 succeeds when key is open but not yet committed + assertDoesNotThrow(() -> { + bucket.rewriteKey("key2", + 1024, + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", "value")); + }); + key2Out.close(); + + // Test 3: After rewrite completes, attempting to rewrite again with -1 fails + key3Out.write("value".getBytes(UTF_8)); + key3Out.close(); + + e = assertThrows(OMException.class, () -> { + bucket.rewriteKey("key2", + 1024, + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", "value")); + }); + + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + assertThat(e).hasMessageContaining("Key already exists"); + } + @ParameterizedTest @EnumSource void cannotRewriteDeletedKey(BucketLayout layout) throws IOException { @@ -4383,6 +4432,12 @@ private void completeMultipartUpload(OzoneBucket bucket, String keyName, assertNotNull(omMultipartUploadCompleteInfo.getHash()); } + private OzoneOutputStream openTestKey(OzoneBucket bucket, String keyName, String keyValue) throws IOException { + return bucket.createKey(keyName, keyValue.getBytes(UTF_8).length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", RandomStringUtils.secure().nextAscii(10))); + } + private OzoneKeyDetails createTestKey(OzoneBucket bucket) throws IOException { return createTestKey(bucket, getTestName(), UUID.randomUUID().toString()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 905b0f9b3a32..652a7aa0fcf1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -280,6 +280,76 @@ public void testAtomicRewrite() throws Exception { assertEquals(acls, committedKey.getAcls()); } + @Test + public void testAtomicCreateIfNotExistsCommitKeyAbsent() throws Exception { + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedDataGeneration(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + assertNull(closedKeyTable.get(getOzonePathKey())); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); + + OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); + assertNotNull(committedKey); + assertNull(committedKey.getExpectedDataGeneration()); + } + + @Test + public void testAtomicCreateIfNotExistsCommitKeyAlreadyExists() throws Exception { + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedDataGeneration(OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + OmKeyInfo existingClosedKey = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())).build(); + closedKeyTable.put(getOzonePathKey(), existingClosedKey); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_ALREADY_EXISTS, omClientResponse.getOMResponse().getStatus()); + } + @Test public void testValidateAndUpdateCacheWithUncommittedBlocks() throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index 1666f4cb38e6..52c9eeea07dd 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -27,6 +27,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.addVolumeAndBucketToDB; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.createOmKeyInfo; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.NOT_A_FILE; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; @@ -146,6 +147,89 @@ public void preExecuteRejectsInvalidReplication() { assertEquals(OMException.ResultCodes.INVALID_REQUEST, e.getResult()); } + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenCreateIfNotExistsKeyMissing( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + + checkResponse(modifiedOmRequest, response, id, false, getBucketLayout()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenCreateIfNotExistsKeyAlreadyExists( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig).setUpdateID(1L).build(); + omMetadataManager.getKeyTable(getBucketLayout()).put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + String openKey = getOpenKey(id); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_ALREADY_EXISTS, response.getOMResponse().getStatus()); + + // As we got error, no entry should be created in openKeyTable. + OmKeyInfo openKeyInfo = + omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + assertNull(openKeyInfo); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateKeyExpectedGenMismatchReturnsKeyGenerationMismatch( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + long expectedGen = 1L; + OMRequest modifiedOmRequest = doPreExecute(createKeyRequest( + false, 0, 100L, replicationConfig, expectedGen)); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig).setUpdateID(2L).build(); + omMetadataManager.getKeyTable(getBucketLayout()).put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + String openKey = getOpenKey(id); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus()); + + // As we got error, no entry should be created in openKeyTable. + OmKeyInfo openKeyInfo = + omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey); + assertNull(openKeyInfo); + } + @ParameterizedTest @MethodSource("data") public void testValidateAndUpdateCache( From 6c8a08ea85050c198c16e5d084bbb57712d98ac4 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 24 Feb 2026 19:30:23 +0800 Subject: [PATCH 13/30] Implement S3 conditional write support --- .../hadoop/ozone/client/OzoneBucket.java | 38 +++++ .../ozone/client/protocol/ClientProtocol.java | 38 +++++ .../hadoop/ozone/client/rpc/RpcClient.java | 62 ++++++++ .../ozone/om/exceptions/OMException.java | 4 + .../hadoop/ozone/om/helpers/OmKeyArgs.java | 16 ++ .../hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +++ .../main/smoketest/s3/conditionalput.robot | 78 ++++++++++ .../ozone/client/rpc/OzoneRpcClientTests.java | 94 ++++++++++++ .../src/main/proto/OmClientProtocol.proto | 14 ++ .../om/request/key/OMKeyCommitRequest.java | 20 +++ .../key/OMKeyCommitRequestWithFSO.java | 1 + .../om/request/key/OMKeyCreateRequest.java | 17 +++ .../ozone/om/request/key/OMKeyRequest.java | 3 + .../request/key/TestOMKeyCommitRequest.java | 137 +++++++++++++++++ .../request/key/TestOMKeyCreateRequest.java | 144 ++++++++++++++++++ .../ozone/s3/endpoint/ObjectEndpoint.java | 74 ++++++++- .../apache/hadoop/ozone/s3/util/S3Consts.java | 4 + .../ozone/client/ClientProtocolStub.java | 19 +++ .../hadoop/ozone/client/OzoneBucketStub.java | 33 ++++ .../ozone/s3/endpoint/TestObjectPut.java | 79 ++++++++++ 20 files changed, 892 insertions(+), 6 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 4b9a3cd0f3d5..89e029a9a9f2 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -510,6 +510,44 @@ public OzoneOutputStream rewriteKey(String keyName, long size, long existingKeyG return proxy.rewriteKey(volumeName, name, keyName, size, existingKeyGeneration, replicationConfig, metadata); } + /** + * Creates a key only if it does not exist (S3 If-None-Match: * semantics). + * + * @param keyName Name of the key + * @param size Size of the data + * @param replicationConfig Replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return OzoneOutputStream to which the data has to be written. + * @throws IOException + */ + public OzoneOutputStream createKeyIfNotExists(String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return proxy.createKeyIfNotExists(volumeName, name, keyName, size, + replicationConfig, metadata, tags); + } + + /** + * Rewrites a key only if its ETag matches (S3 If-Match semantics). + * + * @param keyName Name of the key + * @param size Size of the data + * @param expectedETag The ETag value the existing key must have + * @param replicationConfig Replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return OzoneOutputStream to which the data has to be written. + * @throws IOException + */ + public OzoneOutputStream rewriteKeyIfMatch(String keyName, long size, + String expectedETag, ReplicationConfig replicationConfig, + Map metadata, Map tags) + throws IOException { + return proxy.rewriteKeyIfMatch(volumeName, name, keyName, size, + expectedETag, replicationConfig, metadata, tags); + } + /** * Creates a new key in the bucket, with default replication type RATIS and * with replication factor THREE. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index e3a575896347..19d8f0d75047 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -379,6 +379,44 @@ OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyNam long size, long existingKeyGeneration, ReplicationConfig replicationConfig, Map metadata) throws IOException; + /** + * Creates a key only if it does not exist (S3 If-None-Match: * semantics). + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param replicationConfig The replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneOutputStream} + * @throws OMException with KEY_ALREADY_EXISTS if key exists + */ + OzoneOutputStream createKeyIfNotExists(String volumeName, String bucketName, + String keyName, long size, ReplicationConfig replicationConfig, + Map metadata, Map tags) + throws IOException; + + /** + * Rewrites a key only if its ETag matches (S3 If-Match semantics). + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param expectedETag The ETag value the existing key must have + * @param replicationConfig The replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneOutputStream} + * @throws OMException with ETAG_MISMATCH, ETAG_NOT_AVAILABLE, or KEY_NOT_FOUND + */ + @SuppressWarnings("checkstyle:parameternumber") + OzoneOutputStream rewriteKeyIfMatch(String volumeName, String bucketName, + String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException; + /** * Writes a key in an existing bucket. * @param volumeName Name of the Volume diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 7b1c1a37bf04..39c5c8a50cb0 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1428,6 +1428,68 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String return createOutputStream(openKey); } + @Override + public OzoneOutputStream createKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { + throw new IOException( + "OzoneManager does not support atomic key creation."); + } + + createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); + + OmKeyArgs.Builder builder = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDataSize(size) + .setReplicationConfig(replicationConfig) + .addAllMetadataGdpr(metadata) + .addAllTags(tags) + .setLatestVersionLocation(getLatestVersionLocation) + .setExpectedDataGeneration( + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + + OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); + if (isS3GRequest.get() && size == 0) { + openKey.getKeyInfo().setDataSize(0); + } + return createOutputStream(openKey); + } + + @Override + @SuppressWarnings("checkstyle:parameternumber") + public OzoneOutputStream rewriteKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { + throw new IOException( + "OzoneManager does not support conditional key rewrite."); + } + + createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); + + OmKeyArgs.Builder builder = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDataSize(size) + .setReplicationConfig(replicationConfig) + .addAllMetadataGdpr(metadata) + .addAllTags(tags) + .setLatestVersionLocation(getLatestVersionLocation) + .setExpectedETag(expectedETag); + + OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); + if (isS3GRequest.get() && size == 0) { + openKey.getKeyInfo().setDataSize(0); + } + return createOutputStream(openKey); + } + private void createKeyPreChecks(String volumeName, String bucketName, String keyName, ReplicationConfig replicationConfig) throws IOException { verifyVolumeName(volumeName); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 2b5b5559f92b..dc9d16188c57 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -275,5 +275,9 @@ public enum ResultCodes { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD, TOO_MANY_SNAPSHOTS, + + ETAG_MISMATCH, + + ETAG_NOT_AVAILABLE, } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java index 395425b069e9..da982a568b15 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java @@ -62,6 +62,7 @@ public final class OmKeyArgs implements Auditable { // This allows a key to be created an committed atomically if the original has not // been modified. private Long expectedDataGeneration = null; + private final String expectedETag; private OmKeyArgs(Builder b) { this.volumeName = b.volumeName; @@ -83,6 +84,7 @@ private OmKeyArgs(Builder b) { this.ownerName = b.ownerName; this.tags = b.tags; this.expectedDataGeneration = b.expectedDataGeneration; + this.expectedETag = b.expectedETag; } public boolean getIsMultipartKey() { @@ -169,6 +171,10 @@ public Long getExpectedDataGeneration() { return expectedDataGeneration; } + public String getExpectedETag() { + return expectedETag; + } + @Override public Map toAuditMap() { Map auditMap = new LinkedHashMap<>(); @@ -214,6 +220,9 @@ public KeyArgs toProtobuf() { if (expectedDataGeneration != null) { builder.setExpectedDataGeneration(expectedDataGeneration); } + if (expectedETag != null) { + builder.setExpectedETag(expectedETag); + } return builder.build(); } @@ -240,6 +249,7 @@ public static class Builder { private boolean forceUpdateContainerCacheFromSCM; private final Map tags = new HashMap<>(); private Long expectedDataGeneration = null; + private String expectedETag; public Builder() { this(AclListBuilder.empty()); @@ -267,6 +277,7 @@ public Builder(OmKeyArgs obj) { this.forceUpdateContainerCacheFromSCM = obj.forceUpdateContainerCacheFromSCM; this.expectedDataGeneration = obj.expectedDataGeneration; + this.expectedETag = obj.expectedETag; this.metadata.putAll(obj.metadata); this.tags.putAll(obj.tags); this.acls = AclListBuilder.of(obj.acls); @@ -385,6 +396,11 @@ public Builder setExpectedDataGeneration(long generation) { return this; } + public Builder setExpectedETag(String eTag) { + this.expectedETag = eTag; + return this; + } + public OmKeyArgs build() { return new OmKeyArgs(this); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index b0e26c49d695..09d94ccbd1fe 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -110,6 +110,7 @@ public final class OmKeyInfo extends WithParentObjectId // This allows a key to be created an committed atomically if the original has not // been modified. private Long expectedDataGeneration = null; + private String expectedETag; private OmKeyInfo(Builder b) { super(b); @@ -129,6 +130,7 @@ private OmKeyInfo(Builder b) { this.ownerName = b.ownerName; this.tags = b.tags.build(); this.expectedDataGeneration = b.expectedDataGeneration; + this.expectedETag = b.expectedETag; } private static Codec newCodec(boolean ignorePipeline) { @@ -189,6 +191,14 @@ public Long getExpectedDataGeneration() { return expectedDataGeneration; } + public void setExpectedETag(String eTag) { + this.expectedETag = eTag; + } + + public String getExpectedETag() { + return expectedETag; + } + public String getOwnerName() { return ownerName; } @@ -492,6 +502,7 @@ public static class Builder extends WithParentObjectId.Builder { private boolean isFile; private final MapBuilder tags; private Long expectedDataGeneration = null; + private String expectedETag; public Builder() { this.acls = AclListBuilder.empty(); @@ -514,6 +525,7 @@ public Builder(OmKeyInfo obj) { this.fileChecksum = obj.fileChecksum; this.isFile = obj.isFile; this.expectedDataGeneration = obj.expectedDataGeneration; + this.expectedETag = obj.expectedETag; this.tags = MapBuilder.of(obj.tags); obj.keyLocationVersions.forEach(keyLocationVersion -> this.omKeyLocationInfoGroups.add( @@ -685,6 +697,11 @@ public Builder setExpectedDataGeneration(Long existingGeneration) { return this; } + public Builder setExpectedETag(String eTag) { + this.expectedETag = eTag; + return this; + } + @Override protected void validate() { super.validate(); @@ -804,6 +821,9 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, if (expectedDataGeneration != null) { kb.setExpectedDataGeneration(expectedDataGeneration); } + if (expectedETag != null) { + kb.setExpectedETag(expectedETag); + } if (ownerName != null) { kb.setOwnerName(ownerName); } @@ -857,6 +877,9 @@ public static Builder builderFromProtobuf(KeyInfo keyInfo) { if (keyInfo.hasExpectedDataGeneration()) { builder.setExpectedDataGeneration(keyInfo.getExpectedDataGeneration()); } + if (keyInfo.hasExpectedETag()) { + builder.setExpectedETag(keyInfo.getExpectedETag()); + } if (keyInfo.hasOwnerName()) { builder.setOwnerName(keyInfo.getOwnerName()); diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot new file mode 100644 index 000000000000..11bb0abafc61 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation S3 Conditional Put (If-None-Match / If-Match) tests +Library OperatingSystem +Library String +Library Process +Resource ../commonlib.robot +Resource ./commonawslib.robot +Test Timeout 5 minutes +Suite Setup Setup s3 tests + +*** Variables *** +${ENDPOINT_URL} http://s3g:9878 +${BUCKET} generated + +*** Test Cases *** + +Conditional Put If-None-Match Star Creates New Key + [Documentation] If-None-Match: * should succeed when key does not exist + ${key} = Set Variable condput-ifnonematch-new + Execute echo "test-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match * + Should contain ${result} ETag + +Conditional Put If-None-Match Star Fails For Existing Key + [Documentation] If-None-Match: * should fail with 412 when key already exists + ${key} = Set Variable condput-ifnonematch-existing + Execute echo "initial-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} + Should contain ${result} ETag + # Now try again with If-None-Match: * + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match * + Should contain ${result} PreconditionFailed + +Conditional Put If-Match With Correct ETag Succeeds + [Documentation] If-Match with correct ETag should succeed + ${key} = Set Variable condput-ifmatch-success + Execute echo "initial-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} + Should contain ${result} ETag + # Extract the ETag value + ${etag} = Execute And Ignore Error echo '${result}' | python3 -c "import sys,json; print(json.loads(sys.stdin.read())['ETag'])" + ${etag} = Get From List ${etag} 1 + # Rewrite with matching ETag + Execute echo "updated-content" > /tmp/${key}-updated + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}-updated --if-match ${etag} + Should contain ${result} ETag + +Conditional Put If-Match With Wrong ETag Fails + [Documentation] If-Match with wrong ETag should fail with 412 + ${key} = Set Variable condput-ifmatch-fail + Execute echo "initial-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} + Should contain ${result} ETag + # Try to rewrite with a wrong ETag + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match "wrong-etag" + Should contain ${result} PreconditionFailed + +Conditional Put If-Match On Non-Existent Key Fails + [Documentation] If-Match on a key that does not exist should fail with 412 + ${key} = Set Variable condput-ifmatch-nonexistent + Execute echo "test-content" > /tmp/${key} + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match "some-etag" + Should contain ${result} PreconditionFailed diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 11a831bd7f0f..04952c8e95c4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -1443,6 +1443,100 @@ void cannotRewriteRenamedKey(BucketLayout layout) throws IOException { assertThat(e).hasMessageContaining("not found"); } + @ParameterizedTest + @EnumSource + void testCreateKeyIfNotExistsSuccess(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + String keyName = "create-if-not-exists-" + layout.name(); + byte[] content = "test-content".getBytes(UTF_8); + + try (OzoneOutputStream out = bucket.createKeyIfNotExists( + keyName, content.length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + Collections.emptyMap(), Collections.emptyMap())) { + out.write(content); + } + + assertKeyContent(bucket, keyName, content); + } + + @ParameterizedTest + @EnumSource + void testCreateKeyIfNotExistsFailsWhenKeyExists(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKey(bucket); + byte[] newContent = "new-content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.createKeyIfNotExists( + keyDetails.getName(), newContent.length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + Collections.emptyMap(), Collections.emptyMap())) { + out.write(newContent); + } + }); + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + } + + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchSuccess(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKey(bucket); + String etag = keyDetails.getMetadata().get(OzoneConsts.ETAG); + assertNotNull(etag, "Key should have an ETag"); + + byte[] newContent = "rewritten-content".getBytes(UTF_8); + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + keyDetails.getName(), newContent.length, etag, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + keyDetails.getMetadata(), Collections.emptyMap())) { + out.write(newContent); + } + + assertKeyContent(bucket, keyDetails.getName(), newContent); + } + + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchFailsWithWrongETag(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKey(bucket); + byte[] newContent = "rewritten-content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + keyDetails.getName(), newContent.length, "wrong-etag", + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + keyDetails.getMetadata(), Collections.emptyMap())) { + out.write(newContent); + } + }); + assertEquals(OMException.ResultCodes.ETAG_MISMATCH, e.getResult()); + } + + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchFailsWhenKeyNotFound(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + byte[] content = "content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + "nonexistent-key", content.length, "some-etag", + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + Collections.emptyMap(), Collections.emptyMap())) { + out.write(content); + } + }); + assertEquals(KEY_NOT_FOUND, e.getResult()); + } + private static void rewriteKey( OzoneBucket bucket, OzoneKeyDetails keyDetails, byte[] newContent ) throws IOException { diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index bdb3cc3cee35..057874944670 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -566,6 +566,10 @@ enum Status { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97; TOO_MANY_SNAPSHOTS = 98; + + ETAG_MISMATCH = 99; + + ETAG_NOT_AVAILABLE = 100; } /** @@ -1082,6 +1086,11 @@ message KeyArgs { // This allows a key to be created an committed atomically if the original has not // been modified. optional uint64 expectedDataGeneration = 23; + + // expectedETag, when set, indicates that the existing key must have + // the given ETag for the operation to succeed. This is used for + // S3 conditional writes with the If-Match header. + optional string expectedETag = 24; } message KeyLocation { @@ -1173,6 +1182,11 @@ message KeyInfo { // This allows a key to be created an committed atomically if the original has not // been modified. optional uint64 expectedDataGeneration = 22; + + // expectedETag, when set, indicates that the existing key must have + // the given ETag for the operation to succeed. This is used for + // S3 conditional writes with the If-Match header. + optional string expectedETag = 23; } // KeyInfoProtoLight is a lightweight subset of KeyInfo message containing diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 4d2ce035e9d9..dfbacbc661b4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -303,6 +303,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Set the UpdateID to current transactionLogIndex omKeyInfo = omKeyInfo.toBuilder() .setExpectedDataGeneration(null) + .setExpectedETag(null) .addAllMetadata(KeyValueUtil.getFromProtobuf( commitKeyArgs.getMetadataList())) .setUpdateID(trxnLogIndex) @@ -635,6 +636,25 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map } } } + + if (toCommit.getExpectedETag() != null) { + String expectedETag = toCommit.getExpectedETag(); + auditMap.put("expectedETag", expectedETag); + + if (existing == null) { + throw new OMException("Key not found for If-Match at commit", + OMException.ResultCodes.KEY_NOT_FOUND); + } + String currentETag = existing.getMetadata().get(OzoneConsts.ETAG); + if (currentETag == null) { + throw new OMException("Key does not have an ETag at commit", + OMException.ResultCodes.ETAG_NOT_AVAILABLE); + } + if (!currentETag.equals(expectedETag)) { + throw new OMException("ETag changed during write (concurrent modification)", + OMException.ResultCodes.ETAG_MISMATCH); + } + } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java index 25b5a4b15d41..5aa4fbeb36f4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java @@ -246,6 +246,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Optimistic locking validation has passed. Now set the rewrite fields to null so they are // not persisted in the key table. omKeyInfo.setExpectedDataGeneration(null); + omKeyInfo.setExpectedETag(null); long correctedSpace = omKeyInfo.getReplicatedSize(); // if keyToDelete isn't null, usedNamespace shouldn't check and increase. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 2581195adde4..af766094e40b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -490,5 +490,22 @@ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) } } } + + if (keyArgs.hasExpectedETag()) { + String expectedETag = keyArgs.getExpectedETag(); + if (dbKeyInfo == null) { + throw new OMException("Key not found for If-Match", + OMException.ResultCodes.KEY_NOT_FOUND); + } + String existingETag = dbKeyInfo.getMetadata().get(OzoneConsts.ETAG); + if (existingETag == null) { + throw new OMException("Key does not have an ETag", + OMException.ResultCodes.ETAG_NOT_AVAILABLE); + } + if (!existingETag.equals(expectedETag)) { + throw new OMException("ETag mismatch", + OMException.ResultCodes.ETAG_MISMATCH); + } + } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 7df2619e9e46..13dc904476dd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -993,6 +993,9 @@ protected OmKeyInfo prepareFileInfo( if (keyArgs.hasExpectedDataGeneration()) { builder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); } + if (keyArgs.hasExpectedETag()) { + builder.setExpectedETag(keyArgs.getExpectedETag()); + } return builder.build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 652a7aa0fcf1..1f40b0f72a00 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -350,6 +350,143 @@ public void testAtomicCreateIfNotExistsCommitKeyAlreadyExists() throws Exception assertEquals(KEY_ALREADY_EXISTS, omClientResponse.getOMResponse().getStatus()); } + @Test + public void testCommitWithExpectedETagSuccess() throws Exception { + Table openKeyTable = + omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = + omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = + doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = + getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = + modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + String expectedETag = "matching-etag"; + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedETag(expectedETag); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, + omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + // Add closed key with matching ETag + OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())) + .addMetadata(OzoneConsts.ETAG, expectedETag).build(); + closedKeyTable.put(getOzonePathKey(), closedKeyInfo); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); + + OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); + assertNotNull(committedKey); + assertNull(committedKey.getExpectedETag()); + } + + @Test + public void testCommitWithExpectedETagMismatch() throws Exception { + Table openKeyTable = + omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = + omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = + doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = + getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = + modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedETag("expected-etag"); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, + omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + // Add closed key with different ETag + OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())) + .addMetadata(OzoneConsts.ETAG, "different-etag").build(); + closedKeyTable.put(getOzonePathKey(), closedKeyInfo); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_MISMATCH, + omClientResponse.getOMResponse().getStatus()); + } + + @Test + public void testCommitWithExpectedETagNoETagOnKey() throws Exception { + Table openKeyTable = + omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = + omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = + doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = + getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = + modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedETag("expected-etag"); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, + omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + // Add closed key WITHOUT ETag + OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())).build(); + closedKeyTable.put(getOzonePathKey(), closedKeyInfo); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_NOT_AVAILABLE, + omClientResponse.getOMResponse().getStatus()); + } + @Test public void testValidateAndUpdateCacheWithUncommittedBlocks() throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index 52c9eeea07dd..f9364d87710d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -230,6 +230,126 @@ public void testCreateKeyExpectedGenMismatchReturnsKeyGenerationMismatch( assertNull(openKeyInfo); } + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagKeyNotFound( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag("some-etag")); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagNoETagOnKey( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag("some-etag")); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + // Create existing key without ETag metadata + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig) + .setUpdateID(1L).build(); + omMetadataManager.getKeyTable(getBucketLayout()) + .put(getOzoneKey(), existingKeyInfo); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_NOT_AVAILABLE, + response.getOMResponse().getStatus()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagMismatch( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag("expected-etag")); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + // Create existing key with a different ETag + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig) + .setUpdateID(1L) + .addMetadata(OzoneConsts.ETAG, "different-etag") + .build(); + omMetadataManager.getKeyTable(getBucketLayout()) + .put(getOzoneKey(), existingKeyInfo); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_MISMATCH, + response.getOMResponse().getStatus()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagSuccess( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + String expectedETag = "matching-etag"; + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag(expectedETag)); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + // Create existing key with matching ETag + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig) + .setUpdateID(1L) + .addMetadata(OzoneConsts.ETAG, expectedETag) + .build(); + omMetadataManager.getKeyTable(getBucketLayout()) + .put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, response.getOMResponse().getStatus()); + + // Verify open key was created with expectedETag + OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) + .get(getOpenKey(id)); + assertNotNull(openKeyInfo); + assertEquals(expectedETag, openKeyInfo.getExpectedETag()); + // Creation time should remain the same on rewrite + assertEquals(existingKeyInfo.getCreationTime(), + openKeyInfo.getCreationTime()); + } + @ParameterizedTest @MethodSource("data") public void testValidateAndUpdateCache( @@ -909,6 +1029,30 @@ private OMRequest createKeyRequest( .setCreateKeyRequest(createKeyRequest).build(); } + private OMRequest createKeyRequestWithExpectedETag(String expectedETag) { + KeyArgs.Builder keyArgs = KeyArgs.newBuilder() + .setVolumeName(volumeName).setBucketName(bucketName) + .setKeyName(keyName).setIsMultipartKey(false) + .setFactor( + ((RatisReplicationConfig) replicationConfig) + .getReplicationFactor()) + .setType(replicationConfig.getReplicationType()) + .setLatestVersionLocation(true) + .setDataSize(100L); + + if (expectedETag != null) { + keyArgs.setExpectedETag(expectedETag); + } + + CreateKeyRequest createKeyRequest = + CreateKeyRequest.newBuilder().setKeyArgs(keyArgs).build(); + + return OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.CreateKey) + .setClientId(UUID.randomUUID().toString()) + .setCreateKeyRequest(createKeyRequest).build(); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) public void testKeyCreateWithFileSystemPathsEnabled( diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index d6d9b101c523..9aa7a5ed463c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -188,8 +188,18 @@ public Response put( throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex); } else if (ex.getResult() == ResultCodes.FILE_ALREADY_EXISTS) { throw newError(S3ErrorTable.NO_OVERWRITE, keyPath, ex); + } else if (ex.getResult() == ResultCodes.KEY_ALREADY_EXISTS) { + throw newError(PRECOND_FAILED, keyPath, ex); + } else if (ex.getResult() == ResultCodes.ETAG_MISMATCH) { + throw newError(PRECOND_FAILED, keyPath, ex); + } else if (ex.getResult() == ResultCodes.ETAG_NOT_AVAILABLE) { + throw newError(PRECOND_FAILED, keyPath, ex); } else if (ex.getResult() == ResultCodes.INVALID_REQUEST) { throw newError(S3ErrorTable.INVALID_REQUEST, keyPath); + } else if (ex.getResult() == ResultCodes.KEY_NOT_FOUND + && getHeaders().getHeaderString(S3Consts.IF_MATCH_HEADER) != null) { + // If-Match failed because the key doesn't exist + throw newError(PRECOND_FAILED, keyPath, ex); } else if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) { throw newError(S3ErrorTable.NO_SUCH_KEY, keyPath); } else if (ex.getResult() == ResultCodes.NOT_SUPPORTED_OPERATION) { @@ -231,7 +241,6 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr copyHeader = getHeaders().getHeaderString(COPY_SOURCE_HEADER); - // Normal put object ReplicationConfig replicationConfig = getReplicationConfig(bucket); boolean enableEC = false; @@ -272,6 +281,15 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr return Response.ok().status(HttpStatus.SC_OK).build(); } + String ifNoneMatch = getHeaders().getHeaderString( + S3Consts.IF_NONE_MATCH_HEADER); + String ifMatch = getHeaders().getHeaderString( + S3Consts.IF_MATCH_HEADER); + + if (ifNoneMatch != null && ifMatch != null) { + throw newError(INVALID_REQUEST, keyPath); + } + // Normal put object S3ChunkInputStreamInfo chunkInputStreamInfo = getS3ChunkInputStreamInfo(body, length, amzDecodedLength, keyPath); @@ -282,21 +300,24 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr getCustomMetadataFromHeaders(getHeaders().getRequestHeaders()); Map tags = getTaggingFromHeaders(getHeaders()); + boolean hasConditionalHeaders = ifNoneMatch != null || ifMatch != null; long putLength; final String md5Hash; - if (isDatastreamEnabled() && !enableEC && length > getDatastreamMinLength()) { + if (isDatastreamEnabled() && !enableEC + && length > getDatastreamMinLength() && !hasConditionalHeaders) { perf.appendStreamMode(); Pair keyWriteResult = ObjectEndpointStreaming .put(bucket, keyPath, length, replicationConfig, getChunkSize(), - customMetadata, tags, multiDigestInputStream, getHeaders(), signatureInfo.isSignPayload(), perf); + customMetadata, tags, multiDigestInputStream, getHeaders(), + signatureInfo.isSignPayload(), perf); md5Hash = keyWriteResult.getKey(); putLength = keyWriteResult.getValue(); } else { final String amzContentSha256Header = validateSignatureHeader(getHeaders(), keyPath, signatureInfo.isSignPayload()); - try (OzoneOutputStream output = getClientProtocol().createKey( - volume.getName(), bucketName, keyPath, length, replicationConfig, - customMetadata, tags)) { + try (OzoneOutputStream output = openKeyForPut( + volume.getName(), bucketName, bucket, keyPath, length, + replicationConfig, customMetadata, tags, ifNoneMatch, ifMatch)) { long metadataLatencyNs = getMetrics().updatePutKeyMetadataStats(startNanos); perf.appendMetaLatencyNanos(metadataLatencyNs); @@ -1302,6 +1323,47 @@ private Response deleteObjectTagging(OzoneVolume volume, String bucketName, Stri return Response.noContent().build(); } + /** + * Opens a key for put, applying conditional write logic based on + * If-None-Match and If-Match headers. + */ + @SuppressWarnings("checkstyle:ParameterNumber") + private OzoneOutputStream openKeyForPut(String volumeName, String bucketName, + OzoneBucket bucket, String keyPath, long length, + ReplicationConfig replicationConfig, Map customMetadata, + Map tags, String ifNoneMatch, String ifMatch) + throws IOException { + if (ifNoneMatch != null && "*".equals(ifNoneMatch.trim())) { + return getClientProtocol().createKeyIfNotExists( + volumeName, bucketName, keyPath, length, replicationConfig, + customMetadata, tags); + } else if (ifMatch != null) { + String expectedETag = parseETag(ifMatch); + return getClientProtocol().rewriteKeyIfMatch( + volumeName, bucketName, keyPath, length, expectedETag, + replicationConfig, customMetadata, tags); + } else { + return getClientProtocol().createKey( + volumeName, bucketName, keyPath, length, replicationConfig, + customMetadata, tags); + } + } + + /** + * Parses an ETag from a conditional header value, removing surrounding + * quotes if present. + */ + static String parseETag(String headerValue) { + if (headerValue == null) { + return null; + } + String etag = headerValue.trim(); + if (etag.startsWith("\"") && etag.endsWith("\"")) { + return etag.substring(1, etag.length() - 1); + } + return etag; + } + /** Request context shared among {@code ObjectOperationHandler}s. */ final class ObjectRequestContext { private final String bucketName; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index 797ca1f36712..e90612cc56ed 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -98,6 +98,10 @@ public final class S3Consts { public static final String CHECKSUM_HEADER = "Content-MD5"; + // Conditional request headers + public static final String IF_NONE_MATCH_HEADER = "If-None-Match"; + public static final String IF_MATCH_HEADER = "If-Match"; + //Never Constructed private S3Consts() { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 739babce1d06..c5d2025b8012 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -249,6 +249,25 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String .rewriteKey(keyName, size, existingKeyGeneration, replicationConfig, metadata); } + @Override + public OzoneOutputStream createKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return getBucket(volumeName, bucketName) + .createKeyIfNotExists(keyName, size, replicationConfig, metadata, tags); + } + + @Override + public OzoneOutputStream rewriteKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return getBucket(volumeName, bucketName) + .rewriteKeyIfMatch(keyName, size, expectedETag, replicationConfig, + metadata, tags); + } + @Override public OzoneInputStream getKey(String volumeName, String bucketName, String keyName) throws IOException { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 8037f65fda1c..a91030f6b77a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -192,6 +192,39 @@ public void close() throws IOException { return new OzoneOutputStream(byteArrayOutputStream, null); } + @Override + public OzoneOutputStream createKeyIfNotExists(String keyName, long size, + ReplicationConfig rConfig, Map metadata, + Map tags) throws IOException { + if (keyDetails.containsKey(keyName)) { + throw new OMException("Key already exists", + ResultCodes.KEY_ALREADY_EXISTS); + } + return createKey(keyName, size, rConfig, metadata, tags); + } + + @Override + public OzoneOutputStream rewriteKeyIfMatch(String keyName, long size, + String expectedETag, ReplicationConfig rConfig, + Map metadata, Map tags) + throws IOException { + OzoneKeyDetails existing = keyDetails.get(keyName); + if (existing == null) { + throw new OMException("Key not found for If-Match", + ResultCodes.KEY_NOT_FOUND); + } + String currentETag = existing.getMetadata().get(ETAG); + if (currentETag == null) { + throw new OMException("Key does not have an ETag", + ResultCodes.ETAG_NOT_AVAILABLE); + } + if (!currentETag.equals(expectedETag)) { + throw new OMException("ETag mismatch", + ResultCodes.ETAG_MISMATCH); + } + return createKey(keyName, size, rConfig, metadata, tags); + } + @Override public OzoneDataStreamOutput createStreamKey(String key, long size, ReplicationConfig rConfig, diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index c2456dd068fd..e9d1263a3797 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -555,6 +555,85 @@ private HttpHeaders newMockHttpHeaders() { return httpHeaders; } + @Test + void testIfNoneMatchKeyDoesNotExistSuccess() throws Exception { + when(headers.getHeaderString("If-None-Match")).thenReturn("*"); + + assertSucceeds(() -> putObject(CONTENT)); + assertKeyContent(bucket, KEY_NAME, CONTENT); + } + + @Test + void testIfNoneMatchKeyExistsPreconditionFailed() throws Exception { + // First create the key + assertSucceeds(() -> putObject(CONTENT)); + + // Now try to create again with If-None-Match: * + when(headers.getHeaderString("If-None-Match")).thenReturn("*"); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.PRECOND_FAILED, () -> putObject(CONTENT)); + assertNotNull(ex); + } + + @Test + void testIfMatchETagMatchesSuccess() throws Exception { + // First create the key to get an ETag + Response response = putObject(CONTENT); + String etag = response.getHeaderString(HttpHeaders.ETAG); + assertNotNull(etag); + + // Strip quotes from the ETag + String rawEtag = etag.replace("\"", ""); + + // Now try to rewrite with matching ETag + when(headers.getHeaderString("If-Match")).thenReturn(etag); + + assertSucceeds(() -> putObject("new-content")); + assertKeyContent(bucket, KEY_NAME, "new-content"); + } + + @Test + void testIfMatchETagMismatchPreconditionFailed() throws Exception { + // First create the key + assertSucceeds(() -> putObject(CONTENT)); + + // Try to rewrite with wrong ETag + when(headers.getHeaderString("If-Match")).thenReturn("\"wrong-etag\""); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.PRECOND_FAILED, () -> putObject("new-content")); + assertNotNull(ex); + } + + @Test + void testIfMatchKeyNotFoundPreconditionFailed() throws Exception { + // Try If-Match on a non-existent key + when(headers.getHeaderString("If-Match")).thenReturn("\"some-etag\""); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.PRECOND_FAILED, () -> putObject(CONTENT)); + assertNotNull(ex); + } + + @Test + void testBothHeadersProvidedInvalidRequest() throws Exception { + when(headers.getHeaderString("If-None-Match")).thenReturn("*"); + when(headers.getHeaderString("If-Match")).thenReturn("\"some-etag\""); + + OS3Exception ex = assertErrorResponse( + INVALID_REQUEST, () -> putObject(CONTENT)); + assertNotNull(ex); + } + + @Test + void testParseETag() { + assertEquals("abc123", ObjectEndpoint.parseETag("\"abc123\"")); + assertEquals("abc123", ObjectEndpoint.parseETag("abc123")); + assertEquals("abc123", ObjectEndpoint.parseETag(" \"abc123\" ")); + assertEquals(null, ObjectEndpoint.parseETag(null)); + } + /** Put object at {@code bucketName}/{@code keyName} with pre-defined {@link #CONTENT}. */ private Response putObject(String bucketName, String keyName) throws IOException, OS3Exception { return put(objectEndpoint, bucketName, keyName, CONTENT); From 57d77882c94b59591ddb862e22e00afff10812db Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 13 Mar 2026 01:09:49 +0800 Subject: [PATCH 14/30] Add aws kit tests for S3 conditional object put --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 78 +++++++++++++++++++ .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 73 +++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 7fb2f885725e..379ec2749c13 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -377,6 +377,84 @@ public void testPutObject() { assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); } + @Test + public void testPutObjectIfNoneMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader("If-None-Match", "*"); + + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, metadata); + assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); + } + + @Test + public void testPutObjectIfNoneMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + + InputStream is2 = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader("If-None-Match", "*"); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.putObject(bucketName, keyName, is2, metadata)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(412, ase.getStatusCode()); + assertEquals("PreconditionFailed", ase.getErrorCode()); + } + + @Test + public void testPutObjectIfMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + String etag = putObjectResult.getETag(); + + InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader("If-Match", etag); + + PutObjectResult putObjectResult2 = s3Client.putObject(bucketName, keyName, is2, metadata); + assertNotNull(putObjectResult2.getETag()); + } + + @Test + public void testPutObjectIfMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + + InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader("If-Match", "wrong-etag"); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.putObject(bucketName, keyName, is2, metadata)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(412, ase.getStatusCode()); + assertEquals("PreconditionFailed", ase.getErrorCode()); + } + @Test public void testPutObjectWithMD5Header() throws Exception { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 175dcea265f8..4537bcccdfa8 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -236,6 +236,79 @@ public void testPutObject() { assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", getObjectResponse.eTag()); } + @Test + public void testPutObjectIfNoneMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + PutObjectResponse putObjectResponse = s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifNoneMatch("*"), + RequestBody.fromString(content)); + + assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", putObjectResponse.eTag()); + } + + @Test + public void testPutObjectIfNoneMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content)); + + S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifNoneMatch("*"), + RequestBody.fromString(content))); + + assertEquals(412, exception.statusCode()); + assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode()); + } + + @Test + public void testPutObjectIfMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + PutObjectResponse initialResponse = s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + + PutObjectResponse putObjectResponse = s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifMatch(initialResponse.eTag()), + RequestBody.fromString("bar2")); + + assertNotNull(putObjectResponse.eTag()); + } + + @Test + public void testPutObjectIfMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content)); + + S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifMatch("wrong-etag"), + RequestBody.fromString("bar2"))); + + assertEquals(412, exception.statusCode()); + assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode()); + } + @Test public void testPutObjectEmpty() throws Exception { final String bucketName = getBucketName(); From 6d85ee15b148a57affbe23256b0e1a2aaaf386a0 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 25 Mar 2026 14:10:03 +0800 Subject: [PATCH 15/30] Add tests for S3 object put with conditional headers --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 19 ++++++++++++ .../ozone/s3/endpoint/TestObjectPut.java | 30 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 379ec2749c13..ac51c147e7c6 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -455,6 +455,25 @@ public void testPutObjectIfMatchFail() { assertEquals("PreconditionFailed", ase.getErrorCode()); } + @Test + public void testPutObjectIfMatchMissingKeyFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream("bar2".getBytes( + StandardCharsets.UTF_8)); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader("If-Match", "some-etag"); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.putObject(bucketName, keyName, is, metadata)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(404, ase.getStatusCode()); + assertEquals("NoSuchKey", ase.getErrorCode()); + } + @Test public void testPutObjectWithMD5Header() throws Exception { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index e9d1263a3797..863b38ebc81a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -616,6 +616,36 @@ void testIfMatchKeyNotFoundPreconditionFailed() throws Exception { assertNotNull(ex); } + @Test + void testCopyObjectIfNoneMatchKeyExistsPreconditionFailed() throws Exception { + assertSucceeds(() -> putObject(CONTENT)); + assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, "existing-destination")); + + when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn( + BUCKET_NAME + "/" + urlEncode(KEY_NAME)); + when(headers.getHeaderString("If-None-Match")).thenReturn("*"); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.PRECOND_FAILED, + () -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT)); + assertNotNull(ex); + assertKeyContent(destBucket, DEST_KEY, "existing-destination"); + } + + @Test + void testCopyObjectIfMatchKeyNotFoundNoSuchKey() throws Exception { + assertSucceeds(() -> putObject(CONTENT)); + + when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn( + BUCKET_NAME + "/" + urlEncode(KEY_NAME)); + when(headers.getHeaderString("If-Match")).thenReturn("\"some-etag\""); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.NO_SUCH_KEY, + () -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT)); + assertNotNull(ex); + } + @Test void testBothHeadersProvidedInvalidRequest() throws Exception { when(headers.getHeaderString("If-None-Match")).thenReturn("*"); From a1898c042df76276b412c62e6bd9d7b53373bcb7 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 25 Mar 2026 22:18:42 +0800 Subject: [PATCH 16/30] Add ATOMIC_CREATE_IF_NOT_EXISTS version gate --- .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../hadoop/ozone/OzoneManagerVersion.java | 3 + .../hadoop/ozone/client/OzoneBucket.java | 2 +- .../ozone/client/protocol/ClientProtocol.java | 2 +- .../hadoop/ozone/client/rpc/RpcClient.java | 3 + ...ManagerProtocolClientSideTranslatorPB.java | 2 +- .../ozone/client/rpc/OzoneRpcClientTests.java | 84 +++++++++---------- .../om/request/key/OMKeyCommitRequest.java | 8 +- .../om/request/key/OMKeyCreateRequest.java | 8 +- 9 files changed, 62 insertions(+), 51 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 0e8ea9d677fb..151022823a12 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -318,6 +318,7 @@ public final class OzoneConsts { public static final String TENANT = "tenant"; public static final String USER_PREFIX = "userPrefix"; public static final String REWRITE_GENERATION = "rewriteGeneration"; + /** Sentinel generation used to request atomic create-if-not-exists(put if absent) semantics. */ public static final long EXPECTED_GEN_CREATE_IF_NOT_EXISTS = -1L; public static final String FROM_SNAPSHOT = "fromSnapshot"; public static final String TO_SNAPSHOT = "toSnapshot"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index 41cf8ab28560..7d3f8629f0eb 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -54,6 +54,9 @@ public enum OzoneManagerVersion implements ComponentVersion { S3_LIST_MULTIPART_UPLOADS_PAGINATION(11, "OzoneManager version that supports S3 list multipart uploads API with pagination"), + + ATOMIC_CREATE_IF_NOT_EXISTS(12, + "OzoneManager version that supports explicit create-if-not-exists key semantics"), FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 6aa8ec15b72c..09cd189e8e48 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -508,7 +508,7 @@ public OzoneOutputStream createKey(String key, long size, * * @param keyName Existing key to rewrite. This must exist in the bucket. * @param size The size of the new key - * @param existingKeyGeneration The generation of the existing key which is checked for changes at key create + * @param existingKeyGeneration The positive generation of the existing key which is checked for changes at key create * and commit time. * @param replicationConfig The replication configuration for the key to be rewritten. * @param metadata custom key value metadata diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index e3a575896347..24ddd782cd27 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -368,7 +368,7 @@ OzoneOutputStream createKey(String volumeName, String bucketName, * @param bucketName Name of the Bucket * @param keyName Existing key to rewrite. This must exist in the bucket. * @param size The size of the new key - * @param existingKeyGeneration The generation of the existing key which is checked for changes at key create + * @param existingKeyGeneration The positive generation of the existing key which is checked for changes at key create * and commit time. * @param replicationConfig The replication configuration for the key to be rewritten. * @param metadata custom key value metadata diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 5a35d3a20d55..641a63a28f93 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1408,6 +1408,9 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { throw new IOException("OzoneManager does not support atomic key rewrite."); } + Preconditions.checkArgument(existingKeyGeneration > 0, + "existingKeyGeneration must be positive, but was %s", + existingKeyGeneration); createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 6960c11aaaa7..20f0babab82b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1255,7 +1255,7 @@ public String createSnapshot(String volumeName, if (!StringUtils.isBlank(snapshotName)) { requestBuilder.setSnapshotName(snapshotName); } - + final OMRequest omRequest = createOMRequest(Type.CreateSnapshot) .setCreateSnapshotRequest(requestBuilder) .build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 80da700fdd8d..fa2348127a48 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -1439,50 +1439,22 @@ void rewriteFailsDueToOutdatedGenerationAtCommit(BucketLayout layout) throws IOE @ParameterizedTest @EnumSource - void rewriteFailsWhenKeyExists(BucketLayout layout) throws IOException { + void rewriteRejectsNonPositiveGeneration(BucketLayout layout) + throws IOException { checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); OzoneBucket bucket = createBucket(layout); OzoneKeyDetails key1Details = createTestKey(bucket, "key1", "value".getBytes(UTF_8)); - OzoneOutputStream key2Out = openTestKey(bucket, "key2", "value"); - OzoneOutputStream key3Out = openTestKey(bucket, "key3", "value"); - - // Test 1: Rewrite with -1 fails when key is already committed - OMException e = assertThrows(OMException.class, () -> { - bucket.rewriteKey( - key1Details.getName(), - key1Details.getDataSize(), - EXPECTED_GEN_CREATE_IF_NOT_EXISTS, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), - key1Details.getMetadata()); - }); - - assertEquals(KEY_ALREADY_EXISTS, e.getResult()); - assertThat(e).hasMessageContaining("Key already exists"); - - // Test 2: Rewrite with -1 succeeds when key is open but not yet committed - assertDoesNotThrow(() -> { - bucket.rewriteKey("key2", - 1024, - EXPECTED_GEN_CREATE_IF_NOT_EXISTS, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), - singletonMap("key", "value")); - }); - key2Out.close(); - - // Test 3: After rewrite completes, attempting to rewrite again with -1 fails - key3Out.write("value".getBytes(UTF_8)); - key3Out.close(); - - e = assertThrows(OMException.class, () -> { - bucket.rewriteKey("key2", - 1024, - EXPECTED_GEN_CREATE_IF_NOT_EXISTS, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), - singletonMap("key", "value")); - }); + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> { + bucket.rewriteKey("key2", + 1024, + EXPECTED_GEN_CREATE_IF_NOT_EXISTS, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + singletonMap("key", "value")); + }); - assertEquals(KEY_ALREADY_EXISTS, e.getResult()); - assertThat(e).hasMessageContaining("Key already exists"); + assertThat(e).hasMessageContaining("existingKeyGeneration must be positive"); + assertKeyContent(bucket, key1Details.getName(), "value".getBytes(UTF_8)); } @ParameterizedTest @@ -4500,12 +4472,6 @@ private void completeMultipartUpload(OzoneBucket bucket, String keyName, assertNotNull(omMultipartUploadCompleteInfo.getHash()); } - private OzoneOutputStream openTestKey(OzoneBucket bucket, String keyName, String keyValue) throws IOException { - return bucket.createKey(keyName, keyValue.getBytes(UTF_8).length, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), - singletonMap("key", RandomStringUtils.secure().nextAscii(10))); - } - private OzoneKeyDetails createTestKey(OzoneBucket bucket) throws IOException { return createTestKey(bucket, getTestName(), UUID.randomUUID().toString()); } @@ -4530,6 +4496,32 @@ private OzoneKeyDetails createTestKey( return key; } + private OzoneKeyDetails createTestKeyWithETag(OzoneBucket bucket) + throws IOException { + String keyName = getTestName(); + byte[] bytes = UUID.randomUUID().toString().getBytes(UTF_8); + RatisReplicationConfig replication = + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE); + Map metadata = metadataWithETag( + singletonMap("key", RandomStringUtils.secure().nextAscii(10)), bytes); + try (OzoneOutputStream out = bucket.createKey(keyName, bytes.length, + replication, metadata)) { + out.write(bytes); + } + OzoneKeyDetails key = bucket.getKey(keyName); + assertNotNull(key); + assertEquals(keyName, key.getName()); + assertEquals(DigestUtils.md5Hex(bytes), key.getMetadata().get(ETAG)); + return key; + } + + private static Map metadataWithETag( + Map metadata, byte[] data) { + Map metadataWithETag = new HashMap<>(metadata); + metadataWithETag.put(ETAG, DigestUtils.md5Hex(data)); + return metadataWithETag; + } + private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) { OMException oe = assertThrows(OMException.class, () -> bucket.getKey(keyName)); assertEquals(KEY_NOT_FOUND, oe.getResult()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 4d2ce035e9d9..befa0031b954 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -94,7 +94,13 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { KeyArgs keyArgs = commitKeyRequest.getKeyArgs(); if (keyArgs.hasExpectedDataGeneration()) { - ozoneManager.checkFeatureEnabled(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + if (keyArgs.getExpectedDataGeneration() + == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { + ozoneManager.checkFeatureEnabled( + OzoneManagerVersion.ATOMIC_CREATE_IF_NOT_EXISTS); + } else { + ozoneManager.checkFeatureEnabled(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + } } if (ozoneManager.getConfig().isKeyNameCharacterCheckEnabled()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 2581195adde4..1298ff7426fb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -96,7 +96,13 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { final OMPerformanceMetrics perfMetrics = ozoneManager.getPerfMetrics(); if (keyArgs.hasExpectedDataGeneration()) { - ozoneManager.checkFeatureEnabled(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + if (keyArgs.getExpectedDataGeneration() + == OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS) { + ozoneManager.checkFeatureEnabled( + OzoneManagerVersion.ATOMIC_CREATE_IF_NOT_EXISTS); + } else { + ozoneManager.checkFeatureEnabled(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + } } OmUtils.verifyKeyNameWithSnapshotReservedWord(keyArgs.getKeyName()); From e5f5e3d3408605daf1f6f1844a9293e02c8d5be1 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 25 Mar 2026 22:36:06 +0800 Subject: [PATCH 17/30] suggestion --- .../apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java | 1 - .../hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index fa2348127a48..ff602c8d90ac 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -42,7 +42,6 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index befa0031b954..492d698db997 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -634,9 +634,9 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map if (existing == null) { throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); } - if (!toCommit.getExpectedDataGeneration().equals(existing.getUpdateID())) { + if (expectedGen != existing.getUpdateID()) { throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + - ") does not match the expected generation to rewrite (" + toCommit.getExpectedDataGeneration() + ")", + ") does not match the expected generation to rewrite (" + expectedGen + ")", KEY_NOT_FOUND); } } From 86489b6deb1c6a4f25961e725e305be535b97a56 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 25 Mar 2026 22:43:08 +0800 Subject: [PATCH 18/30] fix pmd --- .../ozone/client/rpc/OzoneRpcClientTests.java | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index ff602c8d90ac..378316c6b15e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -4495,32 +4495,6 @@ private OzoneKeyDetails createTestKey( return key; } - private OzoneKeyDetails createTestKeyWithETag(OzoneBucket bucket) - throws IOException { - String keyName = getTestName(); - byte[] bytes = UUID.randomUUID().toString().getBytes(UTF_8); - RatisReplicationConfig replication = - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE); - Map metadata = metadataWithETag( - singletonMap("key", RandomStringUtils.secure().nextAscii(10)), bytes); - try (OzoneOutputStream out = bucket.createKey(keyName, bytes.length, - replication, metadata)) { - out.write(bytes); - } - OzoneKeyDetails key = bucket.getKey(keyName); - assertNotNull(key); - assertEquals(keyName, key.getName()); - assertEquals(DigestUtils.md5Hex(bytes), key.getMetadata().get(ETAG)); - return key; - } - - private static Map metadataWithETag( - Map metadata, byte[] data) { - Map metadataWithETag = new HashMap<>(metadata); - metadataWithETag.put(ETAG, DigestUtils.md5Hex(data)); - return metadataWithETag; - } - private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) { OMException oe = assertThrows(OMException.class, () -> bucket.getKey(keyName)); assertEquals(KEY_NOT_FOUND, oe.getResult()); From d6395a91f8999018c6907ee08d1dc090e24bbbfd Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 26 Mar 2026 00:28:08 +0800 Subject: [PATCH 19/30] fix pmd --- .../apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java | 6 ------ .../org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java | 3 --- 2 files changed, 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 8b909813af31..e2fcc6051444 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -4566,12 +4566,6 @@ private void completeMultipartUpload(OzoneBucket bucket, String keyName, assertNotNull(omMultipartUploadCompleteInfo.getHash()); } - private OzoneOutputStream openTestKey(OzoneBucket bucket, String keyName, String keyValue) throws IOException { - return bucket.createKey(keyName, keyValue.getBytes(UTF_8).length, - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), - singletonMap("key", RandomStringUtils.secure().nextAscii(10))); - } - private OzoneKeyDetails createTestKey(OzoneBucket bucket) throws IOException { return createTestKey(bucket, getTestName(), UUID.randomUUID().toString()); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index 863b38ebc81a..1a97fd0bb1b1 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -583,9 +583,6 @@ void testIfMatchETagMatchesSuccess() throws Exception { String etag = response.getHeaderString(HttpHeaders.ETAG); assertNotNull(etag); - // Strip quotes from the ETag - String rawEtag = etag.replace("\"", ""); - // Now try to rewrite with matching ETag when(headers.getHeaderString("If-Match")).thenReturn(etag); From c10fb9e0603b1f058bff1769f20c076b52e522ec Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 26 Mar 2026 01:49:42 +0800 Subject: [PATCH 20/30] fix new smoke test --- .../dist/src/main/smoketest/s3/conditionalput.robot | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot index 11bb0abafc61..cac6742f4654 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot @@ -33,7 +33,7 @@ Conditional Put If-None-Match Star Creates New Key [Documentation] If-None-Match: * should succeed when key does not exist ${key} = Set Variable condput-ifnonematch-new Execute echo "test-content" > /tmp/${key} - ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match * + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match "*" Should contain ${result} ETag Conditional Put If-None-Match Star Fails For Existing Key @@ -43,7 +43,7 @@ Conditional Put If-None-Match Star Fails For Existing Key ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} Should contain ${result} ETag # Now try again with If-None-Match: * - ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match * + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match "*" Should contain ${result} PreconditionFailed Conditional Put If-Match With Correct ETag Succeeds @@ -53,8 +53,7 @@ Conditional Put If-Match With Correct ETag Succeeds ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} Should contain ${result} ETag # Extract the ETag value - ${etag} = Execute And Ignore Error echo '${result}' | python3 -c "import sys,json; print(json.loads(sys.stdin.read())['ETag'])" - ${etag} = Get From List ${etag} 1 + ${etag} = Evaluate __import__('json').loads(r'''${result}''')['ETag'].strip('"') # Rewrite with matching ETag Execute echo "updated-content" > /tmp/${key}-updated ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}-updated --if-match ${etag} @@ -67,12 +66,12 @@ Conditional Put If-Match With Wrong ETag Fails ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} Should contain ${result} ETag # Try to rewrite with a wrong ETag - ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match "wrong-etag" + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match wrong-etag Should contain ${result} PreconditionFailed Conditional Put If-Match On Non-Existent Key Fails [Documentation] If-Match on a key that does not exist should fail with 412 ${key} = Set Variable condput-ifmatch-nonexistent Execute echo "test-content" > /tmp/${key} - ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match "some-etag" + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match some-etag Should contain ${result} PreconditionFailed From eea244a9d1f8aa81de99de7d4b50c39deb49393e Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 27 Mar 2026 14:34:08 +0800 Subject: [PATCH 21/30] fix test --- ...ManagerProtocolClientSideTranslatorPB.java | 3 + .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 4 +- .../ozone/client/rpc/OzoneRpcClientTests.java | 55 +++++++++++++++++-- .../ozone/s3/endpoint/TestObjectPut.java | 30 ---------- 4 files changed, 55 insertions(+), 37 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 20f0babab82b..74c88edf0703 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -744,6 +744,9 @@ public OpenKeySession openKey(OmKeyArgs args) throws IOException { if (args.getExpectedDataGeneration() != null) { keyArgs.setExpectedDataGeneration(args.getExpectedDataGeneration()); } + if (args.getExpectedETag() != null) { + keyArgs.setExpectedETag(args.getExpectedETag()); + } req.setKeyArgs(keyArgs.build()); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index ac51c147e7c6..72424c880a24 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -470,8 +470,8 @@ public void testPutObjectIfMatchMissingKeyFail() { () -> s3Client.putObject(bucketName, keyName, is, metadata)); assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(404, ase.getStatusCode()); - assertEquals("NoSuchKey", ase.getErrorCode()); + assertEquals(412, ase.getStatusCode()); + assertEquals("PreconditionFailed", ase.getErrorCode()); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index e2fcc6051444..ec1d522b721e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -1525,19 +1525,24 @@ void testCreateKeyIfNotExistsFailsWhenKeyExists(BucketLayout layout) throws IOEx void testRewriteKeyIfMatchSuccess(BucketLayout layout) throws IOException { checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); OzoneBucket bucket = createBucket(layout); - OzoneKeyDetails keyDetails = createTestKey(bucket); - String etag = keyDetails.getMetadata().get(OzoneConsts.ETAG); + OzoneKeyDetails keyDetails = createTestKeyWithETag(bucket); + String etag = keyDetails.getMetadata().get(ETAG); assertNotNull(etag, "Key should have an ETag"); byte[] newContent = "rewritten-content".getBytes(UTF_8); + Map rewrittenMetadata = new HashMap<>(keyDetails.getMetadata()); + String rewrittenETag = UUID.randomUUID().toString(); + rewrittenMetadata.put(ETAG, rewrittenETag); try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( keyDetails.getName(), newContent.length, etag, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), - keyDetails.getMetadata(), Collections.emptyMap())) { + rewrittenMetadata, Collections.emptyMap())) { out.write(newContent); } assertKeyContent(bucket, keyDetails.getName(), newContent); + assertEquals(rewrittenETag, bucket.getKey(keyDetails.getName()) + .getMetadata().get(ETAG)); } @ParameterizedTest @@ -1545,7 +1550,7 @@ void testRewriteKeyIfMatchSuccess(BucketLayout layout) throws IOException { void testRewriteKeyIfMatchFailsWithWrongETag(BucketLayout layout) throws IOException { checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); OzoneBucket bucket = createBucket(layout); - OzoneKeyDetails keyDetails = createTestKey(bucket); + OzoneKeyDetails keyDetails = createTestKeyWithETag(bucket); byte[] newContent = "rewritten-content".getBytes(UTF_8); OMException e = assertThrows(OMException.class, () -> { @@ -1559,6 +1564,26 @@ void testRewriteKeyIfMatchFailsWithWrongETag(BucketLayout layout) throws IOExcep assertEquals(OMException.ResultCodes.ETAG_MISMATCH, e.getResult()); } + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchFailsWhenETagNotAvailable(BucketLayout layout) + throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKey(bucket); + byte[] newContent = "rewritten-content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + keyDetails.getName(), newContent.length, "some-etag", + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + keyDetails.getMetadata(), Collections.emptyMap())) { + out.write(newContent); + } + }); + assertEquals(OMException.ResultCodes.ETAG_NOT_AVAILABLE, e.getResult()); + } + @ParameterizedTest @EnumSource void testRewriteKeyIfMatchFailsWhenKeyNotFound(BucketLayout layout) throws IOException { @@ -4578,9 +4603,29 @@ private OzoneKeyDetails createTestKey( private OzoneKeyDetails createTestKey( OzoneBucket bucket, String keyName, byte[] bytes + ) throws IOException { + return createTestKey(bucket, keyName, bytes, createTestKeyMetadata()); + } + + private OzoneKeyDetails createTestKeyWithETag(OzoneBucket bucket) + throws IOException { + Map metadata = createTestKeyMetadata(); + metadata.put(ETAG, UUID.randomUUID().toString()); + return createTestKey(bucket, getTestName(), + UUID.randomUUID().toString().getBytes(UTF_8), metadata); + } + + private static Map createTestKeyMetadata() { + Map metadata = new HashMap<>(); + metadata.put("key", RandomStringUtils.secure().nextAscii(10)); + return metadata; + } + + private OzoneKeyDetails createTestKey( + OzoneBucket bucket, String keyName, byte[] bytes, + Map metadata ) throws IOException { RatisReplicationConfig replication = RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE); - Map metadata = singletonMap("key", RandomStringUtils.secure().nextAscii(10)); try (OzoneOutputStream out = bucket.createKey(keyName, bytes.length, replication, metadata)) { out.write(bytes); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index 1a97fd0bb1b1..3c5dca0663ca 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -613,36 +613,6 @@ void testIfMatchKeyNotFoundPreconditionFailed() throws Exception { assertNotNull(ex); } - @Test - void testCopyObjectIfNoneMatchKeyExistsPreconditionFailed() throws Exception { - assertSucceeds(() -> putObject(CONTENT)); - assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, "existing-destination")); - - when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn( - BUCKET_NAME + "/" + urlEncode(KEY_NAME)); - when(headers.getHeaderString("If-None-Match")).thenReturn("*"); - - OS3Exception ex = assertErrorResponse( - S3ErrorTable.PRECOND_FAILED, - () -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT)); - assertNotNull(ex); - assertKeyContent(destBucket, DEST_KEY, "existing-destination"); - } - - @Test - void testCopyObjectIfMatchKeyNotFoundNoSuchKey() throws Exception { - assertSucceeds(() -> putObject(CONTENT)); - - when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn( - BUCKET_NAME + "/" + urlEncode(KEY_NAME)); - when(headers.getHeaderString("If-Match")).thenReturn("\"some-etag\""); - - OS3Exception ex = assertErrorResponse( - S3ErrorTable.NO_SUCH_KEY, - () -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT)); - assertNotNull(ex); - } - @Test void testBothHeadersProvidedInvalidRequest() throws Exception { when(headers.getHeaderString("If-None-Match")).thenReturn("*"); From 5764a70cc97e132744bb2f5a51d254f1738c6764 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 31 Mar 2026 14:58:57 +0800 Subject: [PATCH 22/30] copilot suggestion --- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 6e5e4e0a786e..00f830f4ad0e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -315,7 +315,7 @@ customMetadata, tags, multiDigestInputStream, getHeaders(), final String amzContentSha256Header = validateSignatureHeader(getHeaders(), keyPath, signatureInfo.isSignPayload()); try (OzoneOutputStream output = openKeyForPut( - volume.getName(), bucketName, bucket, keyPath, length, + volume.getName(), bucketName, keyPath, length, replicationConfig, customMetadata, tags, ifNoneMatch, ifMatch)) { long metadataLatencyNs = getMetrics().updatePutKeyMetadataStats(startNanos); @@ -1154,12 +1154,11 @@ private CopyObjectResponse copyObject(OzoneVolume volume, * If-None-Match and If-Match headers. */ @SuppressWarnings("checkstyle:ParameterNumber") - private OzoneOutputStream openKeyForPut(String volumeName, String bucketName, - OzoneBucket bucket, String keyPath, long length, + private OzoneOutputStream openKeyForPut(String volumeName, String bucketName, String keyPath, long length, ReplicationConfig replicationConfig, Map customMetadata, Map tags, String ifNoneMatch, String ifMatch) throws IOException { - if (ifNoneMatch != null && "*".equals(ifNoneMatch.trim())) { + if (ifNoneMatch != null && "*".equals(stripQuotes(ifNoneMatch.trim()))) { return getClientProtocol().createKeyIfNotExists( volumeName, bucketName, keyPath, length, replicationConfig, customMetadata, tags); @@ -1183,11 +1182,7 @@ static String parseETag(String headerValue) { if (headerValue == null) { return null; } - String etag = headerValue.trim(); - if (etag.startsWith("\"") && etag.endsWith("\"")) { - return etag.substring(1, etag.length() - 1); - } - return etag; + return stripQuotes(headerValue.trim()); } /** Request context shared among {@code ObjectOperationHandler}s. */ From e9a45cfc4e8837ac45d86f90d06e439cd4a504fd Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 1 Apr 2026 01:04:17 +0800 Subject: [PATCH 23/30] Use a unified builder to refactor create, rewrite, createIfNotExists and rewriteIfMatch key in RpcClient --- .../hadoop/ozone/client/rpc/RpcClient.java | 127 ++++++------------ .../hadoop/ozone/om/helpers/OmKeyArgs.java | 16 +++ 2 files changed, 60 insertions(+), 83 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index c62927846760..e7636f8077ad 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1369,36 +1369,11 @@ public OzoneOutputStream createKey( String volumeName, String bucketName, String keyName, long size, ReplicationConfig replicationConfig, Map metadata, Map tags) throws IOException { - createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); - - if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) { - if (tags != null && !tags.isEmpty()) { - throw new IOException("OzoneManager does not support object tags"); - } - } - String ownerName = getRealUserInfo().getShortUserName(); - - OmKeyArgs.Builder builder = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(size) - .setReplicationConfig(replicationConfig) - .addAllMetadataGdpr(metadata) - .addAllTags(tags) - .setLatestVersionLocation(getLatestVersionLocation) - .setOwnerName(ownerName); - - OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); - // For bucket with layout OBJECT_STORE, when create an empty file (size=0), - // OM will set DataSize to OzoneConfigKeys#OZONE_SCM_BLOCK_SIZE, - // which will cause S3G's atomic write length check to fail, - // so reset size to 0 here. - if (isS3GRequest.get() && size == 0) { - openKey.getKeyInfo().setDataSize(size); - } - return createOutputStream(openKey); + OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName, + bucketName, keyName, size, replicationConfig, metadata, tags); + builder.setOwnerName(ownerName); + return openOutputStream(builder.build(), size); } @Override @@ -1411,28 +1386,11 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String Preconditions.checkArgument(existingKeyGeneration > 0, "existingKeyGeneration must be positive, but was %s", existingKeyGeneration); - - createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); - - OmKeyArgs.Builder builder = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(size) - .setReplicationConfig(replicationConfig) - .addAllMetadataGdpr(metadata) - .setLatestVersionLocation(getLatestVersionLocation) - .setExpectedDataGeneration(existingKeyGeneration); - - OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); - // For bucket with layout OBJECT_STORE, when create an empty file (size=0), - // OM will set DataSize to OzoneConfigKeys#OZONE_SCM_BLOCK_SIZE, - // which will cause S3G's atomic write length check to fail, - // so reset size to 0 here. - if (isS3GRequest.get() && size == 0) { - openKey.getKeyInfo().setDataSize(0); - } - return createOutputStream(openKey); + OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName, + bucketName, keyName, size, replicationConfig, metadata, + Collections.emptyMap()); + builder.setExpectedDataGeneration(existingKeyGeneration); + return openOutputStream(builder.build(), size); } @Override @@ -1444,26 +1402,11 @@ public OzoneOutputStream createKeyIfNotExists(String volumeName, throw new IOException( "OzoneManager does not support atomic key creation."); } - - createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); - - OmKeyArgs.Builder builder = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(size) - .setReplicationConfig(replicationConfig) - .addAllMetadataGdpr(metadata) - .addAllTags(tags) - .setLatestVersionLocation(getLatestVersionLocation) - .setExpectedDataGeneration( - OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); - - OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); - if (isS3GRequest.get() && size == 0) { - openKey.getKeyInfo().setDataSize(0); - } - return createOutputStream(openKey); + OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName, + bucketName, keyName, size, replicationConfig, metadata, tags); + builder.setExpectedDataGeneration( + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + return openOutputStream(builder.build(), size); } @Override @@ -1476,27 +1419,45 @@ public OzoneOutputStream rewriteKeyIfMatch(String volumeName, throw new IOException( "OzoneManager does not support conditional key rewrite."); } + OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName, + bucketName, keyName, size, replicationConfig, metadata, tags); + builder.setExpectedETag(expectedETag); + return openOutputStream(builder.build(), size); + } + private OmKeyArgs.Builder createWriteKeyArgsBuilder(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) + throws IOException { createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); + validateObjectTagsSupport(tags); + return new OmKeyArgs.Builder(volumeName, bucketName, keyName, size, + replicationConfig, metadata, tags, getLatestVersionLocation); + } - OmKeyArgs.Builder builder = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(size) - .setReplicationConfig(replicationConfig) - .addAllMetadataGdpr(metadata) - .addAllTags(tags) - .setLatestVersionLocation(getLatestVersionLocation) - .setExpectedETag(expectedETag); - - OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); + private OzoneOutputStream openOutputStream(OmKeyArgs keyArgs, long size) + throws IOException { + OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs); + // For bucket with layout OBJECT_STORE, when create an empty file (size=0), + // OM will set DataSize to OzoneConfigKeys#OZONE_SCM_BLOCK_SIZE, + // which will cause S3G's atomic write length check to fail, + // so reset size to 0 here. if (isS3GRequest.get() && size == 0) { openKey.getKeyInfo().setDataSize(0); } return createOutputStream(openKey); } + private void validateObjectTagsSupport(Map tags) + throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) { + if (tags != null && !tags.isEmpty()) { + throw new IOException("OzoneManager does not support object tags"); + } + } + } + private void createKeyPreChecks(String volumeName, String bucketName, String keyName, ReplicationConfig replicationConfig) throws IOException { verifyVolumeName(volumeName); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java index c1054eae98a8..5d2de09c48e5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java @@ -249,6 +249,22 @@ public Builder() { this(AclListBuilder.empty()); } + @SuppressWarnings("checkstyle:parameternumber") + public Builder(String volumeName, String bucketName, String keyName, + long dataSize, ReplicationConfig replicationConfig, + Map metadata, Map tags, + boolean latestVersionLocation) { + this(); + setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDataSize(dataSize) + .setReplicationConfig(replicationConfig) + .addAllMetadataGdpr(metadata) + .addAllTags(tags) + .setLatestVersionLocation(latestVersionLocation); + } + private Builder(AclListBuilder acls) { this.acls = acls; this.tags = MapBuilder.empty(); From e455b887b0699c79a3f19832a4da0c0d5ec22b0d Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 1 Apr 2026 01:24:57 +0800 Subject: [PATCH 24/30] enhance check in robot test and s3 test --- .../main/smoketest/s3/conditionalput.robot | 12 +++++++ .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 17 +++++++++- .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 31 ++++++++++++++++++- 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot index cac6742f4654..93bd25a43e2b 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot @@ -58,6 +58,11 @@ Conditional Put If-Match With Correct ETag Succeeds Execute echo "updated-content" > /tmp/${key}-updated ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}-updated --if-match ${etag} Should contain ${result} ETag + ${new_etag} = Evaluate __import__('json').loads(r'''${result}''')['ETag'].strip('"') + Should Not Be Equal ${new_etag} ${etag} + ${head_result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${key} + ${head_etag} = Evaluate __import__('json').loads(r'''${head_result}''')['ETag'].strip('"') + Should Be Equal ${head_etag} ${new_etag} Conditional Put If-Match With Wrong ETag Fails [Documentation] If-Match with wrong ETag should fail with 412 @@ -65,9 +70,13 @@ Conditional Put If-Match With Wrong ETag Fails Execute echo "initial-content" > /tmp/${key} ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} Should contain ${result} ETag + ${etag} = Evaluate __import__('json').loads(r'''${result}''')['ETag'].strip('"') # Try to rewrite with a wrong ETag ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match wrong-etag Should contain ${result} PreconditionFailed + ${head_result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${key} + ${head_etag} = Evaluate __import__('json').loads(r'''${head_result}''')['ETag'].strip('"') + Should Be Equal ${head_etag} ${etag} Conditional Put If-Match On Non-Existent Key Fails [Documentation] If-Match on a key that does not exist should fail with 412 @@ -75,3 +84,6 @@ Conditional Put If-Match On Non-Existent Key Fails Execute echo "test-content" > /tmp/${key} ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match some-etag Should contain ${result} PreconditionFailed + ${head_result} = Execute AWSS3APICli and ignore error head-object --bucket ${BUCKET} --key ${key} + Should contain ${head_result} 404 + Should contain ${head_result} Not Found diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 72424c880a24..bc81c69f3bd1 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -25,6 +25,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -431,6 +432,10 @@ public void testPutObjectIfMatch() { PutObjectResult putObjectResult2 = s3Client.putObject(bucketName, keyName, is2, metadata); assertNotNull(putObjectResult2.getETag()); + assertNotEquals(etag, putObjectResult2.getETag()); + + ObjectMetadata updatedObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); + assertEquals(putObjectResult2.getETag(), updatedObjectMetadata.getETag()); } @Test @@ -441,7 +446,8 @@ public void testPutObjectIfMatchFail() { s3Client.createBucket(bucketName); InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + PutObjectResult initialResult = + s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); ObjectMetadata metadata = new ObjectMetadata(); @@ -453,6 +459,9 @@ public void testPutObjectIfMatchFail() { assertEquals(ErrorType.Client, ase.getErrorType()); assertEquals(412, ase.getStatusCode()); assertEquals("PreconditionFailed", ase.getErrorCode()); + + ObjectMetadata existingObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); + assertEquals(initialResult.getETag(), existingObjectMetadata.getETag()); } @Test @@ -472,6 +481,12 @@ public void testPutObjectIfMatchMissingKeyFail() { assertEquals(ErrorType.Client, ase.getErrorType()); assertEquals(412, ase.getStatusCode()); assertEquals("PreconditionFailed", ase.getErrorCode()); + + AmazonServiceException missingKey = assertThrows(AmazonServiceException.class, + () -> s3Client.getObject(bucketName, keyName)); + assertEquals(ErrorType.Client, missingKey.getErrorType()); + assertEquals(404, missingKey.getStatusCode()); + assertEquals("NoSuchKey", missingKey.getErrorCode()); } @Test diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 4537bcccdfa8..35a53dd328bb 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -288,6 +289,11 @@ public void testPutObjectIfMatch() { RequestBody.fromString("bar2")); assertNotNull(putObjectResponse.eTag()); + assertNotEquals(initialResponse.eTag(), putObjectResponse.eTag()); + + HeadObjectResponse headObjectResponse = s3Client.headObject( + b -> b.bucket(bucketName).key(keyName)); + assertEquals(putObjectResponse.eTag(), headObjectResponse.eTag()); } @Test @@ -297,7 +303,8 @@ public void testPutObjectIfMatchFail() { final String content = "bar"; s3Client.createBucket(b -> b.bucket(bucketName)); - s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content)); + PutObjectResponse initialResponse = s3Client.putObject( + b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content)); S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b .bucket(bucketName) @@ -307,6 +314,28 @@ public void testPutObjectIfMatchFail() { assertEquals(412, exception.statusCode()); assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode()); + + HeadObjectResponse headObjectResponse = s3Client.headObject( + b -> b.bucket(bucketName).key(keyName)); + assertEquals(initialResponse.eTag(), headObjectResponse.eTag()); + } + + @Test + public void testPutObjectIfMatchMissingKeyFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifMatch("some-etag"), + RequestBody.fromString("bar2"))); + + assertEquals(412, exception.statusCode()); + assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode()); + assertThrows(NoSuchKeyException.class, () -> s3Client.headObject( + b -> b.bucket(bucketName).key(keyName))); } @Test From b496634659cb591e18153b4f84b8ba09af2f80e5 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 1 Apr 2026 14:16:59 +0800 Subject: [PATCH 25/30] allow streaming key to use conditional put request --- .../org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 00f830f4ad0e..de68917f23da 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -299,11 +299,9 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr getCustomMetadataFromHeaders(getHeaders().getRequestHeaders()); Map tags = getTaggingFromHeaders(getHeaders()); - boolean hasConditionalHeaders = ifNoneMatch != null || ifMatch != null; long putLength; final String md5Hash; - if (isDatastreamEnabled() && !enableEC - && length > getDatastreamMinLength() && !hasConditionalHeaders) { + if (isDatastreamEnabled() && !enableEC && length > getDatastreamMinLength()) { perf.appendStreamMode(); Pair keyWriteResult = ObjectEndpointStreaming .put(bucket, keyPath, length, replicationConfig, getChunkSize(), From d929ed993ff3d8c26387373315c6404c198c6d10 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 1 Apr 2026 14:51:30 +0800 Subject: [PATCH 26/30] handle * etag matching --- .../hadoop/ozone/client/OzoneKeyDetails.java | 16 ++++++++++++++++ .../hadoop/ozone/om/helpers/OmKeyInfo.java | 15 +++++++++++++++ .../ozone/om/request/key/OMKeyCommitRequest.java | 5 ++--- .../ozone/om/request/key/OMKeyCreateRequest.java | 5 ++--- .../hadoop/ozone/client/OzoneBucketStub.java | 7 +++---- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index 1bc1418d1f2e..7514f2a7cd71 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.ratis.util.function.CheckedSupplier; @@ -107,4 +108,19 @@ public Long getGeneration() { public OzoneInputStream getContent() throws IOException { return this.contentSupplier.get(); } + + public boolean hasEtag() { + return getMetadata().containsKey(OzoneConsts.ETAG); + } + + public boolean isEtagEquals(String matchingETag) { + String currentETag = getMetadata().get(OzoneConsts.ETAG); + if (currentETag == null) { + return false; + } + if (matchingETag.equals("*")) { + return true; + } + return currentETag.equals(matchingETag); + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index 09d94ccbd1fe..e388df27b4b3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -1020,6 +1020,21 @@ public boolean hasBlocks() { return false; } + public boolean hasEtag() { + return getMetadata().containsKey(OzoneConsts.ETAG); + } + + public boolean isEtagEquals(String matchingETag) { + String currentETag = getMetadata().get(OzoneConsts.ETAG); + if (currentETag == null) { + return false; + } + if (matchingETag.equals("*")) { + return true; + } + return currentETag.equals(matchingETag); + } + public static boolean isKeyEmpty(@Nullable OmKeyInfo keyInfo) { if (keyInfo == null) { return true; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index a6250d35fd30..65b3394b5fcd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -651,12 +651,11 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map throw new OMException("Key not found for If-Match at commit", OMException.ResultCodes.KEY_NOT_FOUND); } - String currentETag = existing.getMetadata().get(OzoneConsts.ETAG); - if (currentETag == null) { + if (!existing.hasEtag()) { throw new OMException("Key does not have an ETag at commit", OMException.ResultCodes.ETAG_NOT_AVAILABLE); } - if (!currentETag.equals(expectedETag)) { + if (!existing.isEtagEquals(expectedETag)) { throw new OMException("ETag changed during write (concurrent modification)", OMException.ResultCodes.ETAG_MISMATCH); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 9a0f7b2af19b..4d37a1d7aa7f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -503,12 +503,11 @@ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) throw new OMException("Key not found for If-Match", OMException.ResultCodes.KEY_NOT_FOUND); } - String existingETag = dbKeyInfo.getMetadata().get(OzoneConsts.ETAG); - if (existingETag == null) { + if (!dbKeyInfo.hasEtag()) { throw new OMException("Key does not have an ETag", OMException.ResultCodes.ETAG_NOT_AVAILABLE); } - if (!existingETag.equals(expectedETag)) { + if (!dbKeyInfo.isEtagEquals(expectedETag)) { throw new OMException("ETag mismatch", OMException.ResultCodes.ETAG_MISMATCH); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index a91030f6b77a..0bd67b45e341 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -104,7 +104,7 @@ public OzoneBucketStub build() { return new OzoneBucketStub(this); } } - + @Override public OzoneOutputStream createKey(String key, long size) throws IOException { return createKey(key, size, @@ -213,12 +213,11 @@ public OzoneOutputStream rewriteKeyIfMatch(String keyName, long size, throw new OMException("Key not found for If-Match", ResultCodes.KEY_NOT_FOUND); } - String currentETag = existing.getMetadata().get(ETAG); - if (currentETag == null) { + if (!existing.hasEtag()) { throw new OMException("Key does not have an ETag", ResultCodes.ETAG_NOT_AVAILABLE); } - if (!currentETag.equals(expectedETag)) { + if (!existing.isEtagEquals(expectedETag)) { throw new OMException("ETag mismatch", ResultCodes.ETAG_MISMATCH); } From d5d74c9c37a49375a4e8e1a1092782488243cf9f Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 1 Apr 2026 16:08:53 +0800 Subject: [PATCH 27/30] set error message when validating the conditional request --- .../ozone/s3/endpoint/ObjectEndpoint.java | 30 ++++++++++++++++-- .../ozone/s3/endpoint/TestObjectPut.java | 31 +++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index de68917f23da..4212dc2dd3df 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -285,8 +285,31 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr String ifMatch = getHeaders().getHeaderString( S3Consts.IF_MATCH_HEADER); - if (ifNoneMatch != null && ifMatch != null) { - throw newError(INVALID_REQUEST, keyPath); + if (ifNoneMatch != null && StringUtils.isBlank(ifNoneMatch)) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage("If-None-Match header cannot be empty."); + throw ex; + } + if (ifMatch != null && StringUtils.isBlank(ifMatch)) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage("If-Match header cannot be empty."); + throw ex; + } + + String ifNoneMatchTrimmed = ifNoneMatch == null ? null : ifNoneMatch.trim(); + String ifMatchTrimmed = ifMatch == null ? null : ifMatch.trim(); + + if (ifNoneMatchTrimmed != null && ifMatchTrimmed != null) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage("If-Match and If-None-Match cannot be specified together."); + throw ex; + } + + if (ifNoneMatchTrimmed != null + && !"*".equals(stripQuotes(ifNoneMatchTrimmed))) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage("Only If-None-Match: * is supported for conditional put."); + throw ex; } // Normal put object @@ -314,7 +337,8 @@ customMetadata, tags, multiDigestInputStream, getHeaders(), validateSignatureHeader(getHeaders(), keyPath, signatureInfo.isSignPayload()); try (OzoneOutputStream output = openKeyForPut( volume.getName(), bucketName, keyPath, length, - replicationConfig, customMetadata, tags, ifNoneMatch, ifMatch)) { + replicationConfig, customMetadata, tags, ifNoneMatchTrimmed, + ifMatchTrimmed)) { long metadataLatencyNs = getMetrics().updatePutKeyMetadataStats(startNanos); perf.appendMetaLatencyNanos(metadataLatencyNs); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index 3c5dca0663ca..e85b121aaaf5 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -621,6 +621,37 @@ void testBothHeadersProvidedInvalidRequest() throws Exception { OS3Exception ex = assertErrorResponse( INVALID_REQUEST, () -> putObject(CONTENT)); assertNotNull(ex); + assertThat(ex.getErrorMessage()).contains( + "If-Match and If-None-Match cannot be specified together"); + } + + @Test + void testBlankIfNoneMatchInvalidRequest() throws Exception { + when(headers.getHeaderString("If-None-Match")).thenReturn(" "); + + OS3Exception ex = assertErrorResponse( + INVALID_REQUEST, () -> putObject(CONTENT)); + assertThat(ex.getErrorMessage()).contains( + "If-None-Match header cannot be empty"); + } + + @Test + void testBlankIfMatchInvalidRequest() throws Exception { + when(headers.getHeaderString("If-Match")).thenReturn(" "); + + OS3Exception ex = assertErrorResponse( + INVALID_REQUEST, () -> putObject(CONTENT)); + assertThat(ex.getErrorMessage()).contains("If-Match header cannot be empty"); + } + + @Test + void testIfNoneMatchNotStarInvalidRequest() throws Exception { + when(headers.getHeaderString("If-None-Match")).thenReturn("\"etag\""); + + OS3Exception ex = assertErrorResponse( + INVALID_REQUEST, () -> putObject(CONTENT)); + assertThat(ex.getErrorMessage()).contains( + "Only If-None-Match: * is supported"); } @Test From 77aa2e629fbdd0621dedabe8edcffba5bc21d7dd Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 1 Apr 2026 16:39:39 +0800 Subject: [PATCH 28/30] use standard sdk request builder to build conditional request in v2 sdk test --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index bc81c69f3bd1..d8fbd1f1ce39 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -386,10 +386,10 @@ public void testPutObjectIfNoneMatch() { s3Client.createBucket(bucketName); InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setHeader("If-None-Match", "*"); + PutObjectRequest request = new PutObjectRequest( + bucketName, keyName, is, new ObjectMetadata()).ifNoneMatch("*"); - PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, metadata); + PutObjectResult putObjectResult = s3Client.putObject(request); assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); } @@ -404,11 +404,11 @@ public void testPutObjectIfNoneMatchFail() { s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); InputStream is2 = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setHeader("If-None-Match", "*"); + PutObjectRequest request = new PutObjectRequest( + bucketName, keyName, is2, new ObjectMetadata()).ifNoneMatch("*"); AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(bucketName, keyName, is2, metadata)); + () -> s3Client.putObject(request)); assertEquals(ErrorType.Client, ase.getErrorType()); assertEquals(412, ase.getStatusCode()); @@ -427,10 +427,10 @@ public void testPutObjectIfMatch() { String etag = putObjectResult.getETag(); InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setHeader("If-Match", etag); + PutObjectRequest request = new PutObjectRequest( + bucketName, keyName, is2, new ObjectMetadata()).ifMatch(etag); - PutObjectResult putObjectResult2 = s3Client.putObject(bucketName, keyName, is2, metadata); + PutObjectResult putObjectResult2 = s3Client.putObject(request); assertNotNull(putObjectResult2.getETag()); assertNotEquals(etag, putObjectResult2.getETag()); @@ -450,11 +450,11 @@ public void testPutObjectIfMatchFail() { s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setHeader("If-Match", "wrong-etag"); + PutObjectRequest request = new PutObjectRequest( + bucketName, keyName, is2, new ObjectMetadata()).ifMatch("wrong-etag"); AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(bucketName, keyName, is2, metadata)); + () -> s3Client.putObject(request)); assertEquals(ErrorType.Client, ase.getErrorType()); assertEquals(412, ase.getStatusCode()); @@ -472,11 +472,11 @@ public void testPutObjectIfMatchMissingKeyFail() { InputStream is = new ByteArrayInputStream("bar2".getBytes( StandardCharsets.UTF_8)); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setHeader("If-Match", "some-etag"); + PutObjectRequest request = new PutObjectRequest( + bucketName, keyName, is, new ObjectMetadata()).ifMatch("some-etag"); AmazonServiceException ase = assertThrows(AmazonServiceException.class, - () -> s3Client.putObject(bucketName, keyName, is, metadata)); + () -> s3Client.putObject(request)); assertEquals(ErrorType.Client, ase.getErrorType()); assertEquals(412, ase.getStatusCode()); From 2ed823d78305f841a1436e63ff248c0d5680d692 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 1 Apr 2026 16:40:56 +0800 Subject: [PATCH 29/30] add conditional put endpoints for streaming --- .../hadoop/ozone/client/OzoneBucket.java | 46 ++++++++++++ .../ozone/client/protocol/ClientProtocol.java | 40 +++++++++++ .../hadoop/ozone/client/rpc/RpcClient.java | 72 +++++++++++++------ .../ozone/s3/endpoint/ObjectEndpoint.java | 3 +- .../s3/endpoint/ObjectEndpointStreaming.java | 31 ++++++-- .../ozone/client/ClientProtocolStub.java | 20 ++++++ .../hadoop/ozone/client/OzoneBucketStub.java | 32 +++++++++ 7 files changed, 216 insertions(+), 28 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 3bfad0ca385b..2bde6939651d 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -613,6 +613,52 @@ public OzoneDataStreamOutput createStreamKey(String key, long size, replicationConfig, keyMetadata, tags); } + /** + * Creates a key with datastream only if it does not exist already + * (S3 If-None-Match: * semantics). + * + * @param key Name of the key to be created. + * @param size Size of the data the key will point to. + * @param replicationConfig Replication configuration. + * @param keyMetadata Custom key metadata. + * @param tags Tags used for S3 object tags + * @return OzoneDataStreamOutput to which the data has to be written. + * @throws IOException + */ + public OzoneDataStreamOutput createStreamKeyIfNotExists(String key, long size, + ReplicationConfig replicationConfig, Map keyMetadata, + Map tags) throws IOException { + if (replicationConfig == null) { + replicationConfig = defaultReplication; + } + return proxy.createStreamKeyIfNotExists(volumeName, name, key, size, + replicationConfig, keyMetadata, tags); + } + + /** + * Rewrites a key with datastream only if its ETag matches + * (S3 If-Match semantics). + * + * @param key Name of the key to be rewritten. + * @param size Size of the data the key will point to. + * @param expectedETag The ETag value the existing key must have. + * @param replicationConfig Replication configuration. + * @param keyMetadata Custom key metadata. + * @param tags Tags used for S3 object tags + * @return OzoneDataStreamOutput to which the data has to be written. + * @throws IOException + */ + public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String key, long size, + String expectedETag, ReplicationConfig replicationConfig, + Map keyMetadata, Map tags) + throws IOException { + if (replicationConfig == null) { + replicationConfig = defaultReplication; + } + return proxy.rewriteStreamKeyIfMatch(volumeName, name, key, size, + expectedETag, replicationConfig, keyMetadata, tags); + } + /** * Reads an existing key from the bucket. * diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index dc128a7bde50..2542fda26e81 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -464,6 +464,46 @@ OzoneDataStreamOutput createStreamKey(String volumeName, String bucketName, Map metadata, Map tags) throws IOException; + /** + * Writes a key in an existing bucket only if it does not already exist + * (S3 If-None-Match: * semantics). + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param replicationConfig The replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneDataStreamOutput} + * @throws OMException with KEY_ALREADY_EXISTS if key exists + */ + OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException; + + /** + * Writes a key in an existing bucket only if its ETag matches + * (S3 If-Match semantics). + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param expectedETag The ETag value the existing key must have + * @param replicationConfig The replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneDataStreamOutput} + * @throws OMException with ETAG_MISMATCH, ETAG_NOT_AVAILABLE, or KEY_NOT_FOUND + */ + @SuppressWarnings("checkstyle:parameternumber") + OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException; + /** * Reads a key from an existing bucket. * @param volumeName Name of the Volume diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index e7636f8077ad..f2440a9623f7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1497,33 +1497,61 @@ public OzoneDataStreamOutput createStreamKey( String volumeName, String bucketName, String keyName, long size, ReplicationConfig replicationConfig, Map metadata, Map tags) throws IOException { - verifyVolumeName(volumeName); - verifyBucketName(bucketName); - if (checkKeyNameEnabled) { - HddsClientUtils.verifyKeyName(keyName); - } - HddsClientUtils.checkNotNull(keyName); + OmKeyArgs.Builder builder = createStreamKeyArgsBuilder( + volumeName, bucketName, keyName, size, replicationConfig, metadata, + tags); + return openDataStreamOutput(builder.build()); + } - if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) { - if (tags != null && !tags.isEmpty()) { - throw new IOException("OzoneManager does not support object tags"); - } + @Override + public OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { + throw new IOException( + "OzoneManager does not support atomic key creation."); } + OmKeyArgs.Builder builder = createStreamKeyArgsBuilder( + volumeName, bucketName, keyName, size, replicationConfig, metadata, + tags); + builder.setExpectedDataGeneration( + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + return openDataStreamOutput(builder.build()); + } - String ownerName = getRealUserInfo().getShortUserName(); + @Override + @SuppressWarnings("checkstyle:parameternumber") + public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { + throw new IOException( + "OzoneManager does not support conditional key rewrite."); + } + OmKeyArgs.Builder builder = createStreamKeyArgsBuilder( + volumeName, bucketName, keyName, size, replicationConfig, metadata, + tags); + builder.setExpectedETag(expectedETag); + return openDataStreamOutput(builder.build()); + } - OmKeyArgs.Builder builder = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(size) - .setReplicationConfig(replicationConfig) - .addAllMetadataGdpr(metadata) - .addAllTags(tags) - .setSortDatanodesInPipeline(true) - .setOwnerName(ownerName); + private OmKeyArgs.Builder createStreamKeyArgsBuilder(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) + throws IOException { + OmKeyArgs.Builder builder = createWriteKeyArgsBuilder( + volumeName, bucketName, keyName, size, replicationConfig, metadata, + tags); + builder.setOwnerName(getRealUserInfo().getShortUserName()); + return builder; + } - OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); + private OzoneDataStreamOutput openDataStreamOutput(OmKeyArgs keyArgs) + throws IOException { + OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs); return createDataStreamOutput(openKey); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 4212dc2dd3df..ad67ebd25908 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -329,7 +329,8 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr Pair keyWriteResult = ObjectEndpointStreaming .put(bucket, keyPath, length, replicationConfig, getChunkSize(), customMetadata, tags, multiDigestInputStream, getHeaders(), - signatureInfo.isSignPayload(), perf); + signatureInfo.isSignPayload(), perf, ifNoneMatchTrimmed, + ifMatchTrimmed); md5Hash = keyWriteResult.getKey(); putLength = keyWriteResult.getValue(); } else { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java index 767c11506dc1..336f80987b7b 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -73,12 +73,13 @@ public static Pair put( int chunkSize, Map keyMetadata, Map tags, MultiDigestInputStream body, HttpHeaders headers, boolean isSignedPayload, - PerformanceStringBuilder perf) + PerformanceStringBuilder perf, String ifNoneMatch, String ifMatch) throws IOException, OS3Exception { try { return putKeyWithStream(bucket, keyPath, - length, chunkSize, replicationConfig, keyMetadata, tags, body, headers, isSignedPayload, perf); + length, chunkSize, replicationConfig, keyMetadata, tags, body, + headers, isSignedPayload, perf, ifNoneMatch, ifMatch); } catch (IOException ex) { LOG.error("Exception occurred in PutObject", ex); if (ex instanceof OMException) { @@ -113,14 +114,17 @@ public static Pair putKeyWithStream( MultiDigestInputStream body, HttpHeaders headers, boolean isSignedPayload, - PerformanceStringBuilder perf) + PerformanceStringBuilder perf, + String ifNoneMatch, + String ifMatch) throws IOException, OS3Exception { long startNanos = Time.monotonicNowNanos(); final String amzContentSha256Header = validateSignatureHeader(headers, keyPath, isSignedPayload); long writeLen; String md5Hash; - try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath, - length, replicationConfig, keyMetadata, tags)) { + try (OzoneDataStreamOutput streamOutput = openStreamKeyForPut(bucket, + keyPath, length, replicationConfig, keyMetadata, tags, ifNoneMatch, + ifMatch)) { long metadataLatencyNs = METRICS.updatePutKeyMetadataStats(startNanos); writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length); md5Hash = DatatypeConverter.printHexBinary(body.getMessageDigest(OzoneConsts.MD5_HASH).digest()) @@ -155,6 +159,23 @@ public static Pair putKeyWithStream( return Pair.of(md5Hash, writeLen); } + private static OzoneDataStreamOutput openStreamKeyForPut(OzoneBucket bucket, + String keyPath, long length, ReplicationConfig replicationConfig, + Map keyMetadata, Map tags, + String ifNoneMatch, String ifMatch) throws IOException { + if (ifNoneMatch != null && "*".equals(ObjectEndpoint.parseETag(ifNoneMatch))) { + return bucket.createStreamKeyIfNotExists(keyPath, length, + replicationConfig, keyMetadata, tags); + } + if (ifMatch != null) { + return bucket.rewriteStreamKeyIfMatch(keyPath, length, + ObjectEndpoint.parseETag(ifMatch), replicationConfig, keyMetadata, + tags); + } + return bucket.createStreamKey(keyPath, length, replicationConfig, + keyMetadata, tags); + } + @SuppressWarnings("checkstyle:ParameterNumber") public static long copyKeyWithStream( OzoneBucket bucket, diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index c5d2025b8012..32a969fddbc0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -268,6 +268,26 @@ public OzoneOutputStream rewriteKeyIfMatch(String volumeName, metadata, tags); } + @Override + public OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return getBucket(volumeName, bucketName) + .createStreamKeyIfNotExists(keyName, size, replicationConfig, + metadata, tags); + } + + @Override + public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return getBucket(volumeName, bucketName) + .rewriteStreamKeyIfMatch(keyName, size, expectedETag, + replicationConfig, metadata, tags); + } + @Override public OzoneInputStream getKey(String volumeName, String bucketName, String keyName) throws IOException { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 0bd67b45e341..75cc73aadf20 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -279,6 +279,38 @@ public void flush() throws IOException { return new OzoneDataStreamOutputStub(byteBufferStreamOutput, key + size); } + @Override + public OzoneDataStreamOutput createStreamKeyIfNotExists(String key, long size, + ReplicationConfig rConfig, Map keyMetadata, + Map tags) throws IOException { + if (keyDetails.containsKey(key)) { + throw new OMException("Key already exists", + ResultCodes.KEY_ALREADY_EXISTS); + } + return createStreamKey(key, size, rConfig, keyMetadata, tags); + } + + @Override + public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String key, long size, + String expectedETag, ReplicationConfig rConfig, + Map keyMetadata, Map tags) + throws IOException { + OzoneKeyDetails existing = keyDetails.get(key); + if (existing == null) { + throw new OMException("Key not found for If-Match", + ResultCodes.KEY_NOT_FOUND); + } + if (!existing.hasEtag()) { + throw new OMException("Key does not have an ETag", + ResultCodes.ETAG_NOT_AVAILABLE); + } + if (!existing.isEtagEquals(expectedETag)) { + throw new OMException("ETag mismatch", + ResultCodes.ETAG_MISMATCH); + } + return createStreamKey(key, size, rConfig, keyMetadata, tags); + } + @Override public OzoneDataStreamOutput createMultipartStreamKey(String key, long size, From 3a5e880a4d3fbfc7b3d631fed7ee151ec1f65801 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 1 Apr 2026 16:50:18 +0800 Subject: [PATCH 30/30] checkstyle --- .../apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java index 336f80987b7b..2eb2dd21f30c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -159,6 +159,7 @@ public static Pair putKeyWithStream( return Pair.of(md5Hash, writeLen); } + @SuppressWarnings("checkstyle:ParameterNumber") private static OzoneDataStreamOutput openStreamKeyForPut(OzoneBucket bucket, String keyPath, long length, ReplicationConfig replicationConfig, Map keyMetadata, Map tags,