diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 09cd189e8e48..2bde6939651d 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -520,6 +520,44 @@ public OzoneOutputStream rewriteKey(String keyName, long size, long existingKeyG return proxy.rewriteKey(volumeName, name, keyName, size, existingKeyGeneration, replicationConfig, metadata); } + /** + * Creates a key only if it does not exist (S3 If-None-Match: * semantics). + * + * @param keyName Name of the key + * @param size Size of the data + * @param replicationConfig Replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return OzoneOutputStream to which the data has to be written. + * @throws IOException + */ + public OzoneOutputStream createKeyIfNotExists(String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return proxy.createKeyIfNotExists(volumeName, name, keyName, size, + replicationConfig, metadata, tags); + } + + /** + * Rewrites a key only if its ETag matches (S3 If-Match semantics). + * + * @param keyName Name of the key + * @param size Size of the data + * @param expectedETag The ETag value the existing key must have + * @param replicationConfig Replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return OzoneOutputStream to which the data has to be written. + * @throws IOException + */ + public OzoneOutputStream rewriteKeyIfMatch(String keyName, long size, + String expectedETag, ReplicationConfig replicationConfig, + Map metadata, Map tags) + throws IOException { + return proxy.rewriteKeyIfMatch(volumeName, name, keyName, size, + expectedETag, replicationConfig, metadata, tags); + } + /** * Creates a new key in the bucket, with default replication type RATIS and * with replication factor THREE. @@ -575,6 +613,52 @@ public OzoneDataStreamOutput createStreamKey(String key, long size, replicationConfig, keyMetadata, tags); } + /** + * Creates a key with datastream only if it does not exist already + * (S3 If-None-Match: * semantics). + * + * @param key Name of the key to be created. + * @param size Size of the data the key will point to. + * @param replicationConfig Replication configuration. + * @param keyMetadata Custom key metadata. + * @param tags Tags used for S3 object tags + * @return OzoneDataStreamOutput to which the data has to be written. + * @throws IOException + */ + public OzoneDataStreamOutput createStreamKeyIfNotExists(String key, long size, + ReplicationConfig replicationConfig, Map keyMetadata, + Map tags) throws IOException { + if (replicationConfig == null) { + replicationConfig = defaultReplication; + } + return proxy.createStreamKeyIfNotExists(volumeName, name, key, size, + replicationConfig, keyMetadata, tags); + } + + /** + * Rewrites a key with datastream only if its ETag matches + * (S3 If-Match semantics). + * + * @param key Name of the key to be rewritten. + * @param size Size of the data the key will point to. + * @param expectedETag The ETag value the existing key must have. + * @param replicationConfig Replication configuration. + * @param keyMetadata Custom key metadata. + * @param tags Tags used for S3 object tags + * @return OzoneDataStreamOutput to which the data has to be written. + * @throws IOException + */ + public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String key, long size, + String expectedETag, ReplicationConfig replicationConfig, + Map keyMetadata, Map tags) + throws IOException { + if (replicationConfig == null) { + replicationConfig = defaultReplication; + } + return proxy.rewriteStreamKeyIfMatch(volumeName, name, key, size, + expectedETag, replicationConfig, keyMetadata, tags); + } + /** * Reads an existing key from the bucket. * diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index 1bc1418d1f2e..7514f2a7cd71 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.ratis.util.function.CheckedSupplier; @@ -107,4 +108,19 @@ public Long getGeneration() { public OzoneInputStream getContent() throws IOException { return this.contentSupplier.get(); } + + public boolean hasEtag() { + return getMetadata().containsKey(OzoneConsts.ETAG); + } + + public boolean isEtagEquals(String matchingETag) { + String currentETag = getMetadata().get(OzoneConsts.ETAG); + if (currentETag == null) { + return false; + } + if (matchingETag.equals("*")) { + return true; + } + return currentETag.equals(matchingETag); + } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 24ddd782cd27..2542fda26e81 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -379,6 +379,44 @@ OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyNam long size, long existingKeyGeneration, ReplicationConfig replicationConfig, Map metadata) throws IOException; + /** + * Creates a key only if it does not exist (S3 If-None-Match: * semantics). + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param replicationConfig The replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneOutputStream} + * @throws OMException with KEY_ALREADY_EXISTS if key exists + */ + OzoneOutputStream createKeyIfNotExists(String volumeName, String bucketName, + String keyName, long size, ReplicationConfig replicationConfig, + Map metadata, Map tags) + throws IOException; + + /** + * Rewrites a key only if its ETag matches (S3 If-Match semantics). + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param expectedETag The ETag value the existing key must have + * @param replicationConfig The replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneOutputStream} + * @throws OMException with ETAG_MISMATCH, ETAG_NOT_AVAILABLE, or KEY_NOT_FOUND + */ + @SuppressWarnings("checkstyle:parameternumber") + OzoneOutputStream rewriteKeyIfMatch(String volumeName, String bucketName, + String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException; + /** * Writes a key in an existing bucket. * @param volumeName Name of the Volume @@ -426,6 +464,46 @@ OzoneDataStreamOutput createStreamKey(String volumeName, String bucketName, Map metadata, Map tags) throws IOException; + /** + * Writes a key in an existing bucket only if it does not already exist + * (S3 If-None-Match: * semantics). + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param replicationConfig The replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneDataStreamOutput} + * @throws OMException with KEY_ALREADY_EXISTS if key exists + */ + OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException; + + /** + * Writes a key in an existing bucket only if its ETag matches + * (S3 If-Match semantics). + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param expectedETag The ETag value the existing key must have + * @param replicationConfig The replication configuration + * @param metadata custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneDataStreamOutput} + * @throws OMException with ETAG_MISMATCH, ETAG_NOT_AVAILABLE, or KEY_NOT_FOUND + */ + @SuppressWarnings("checkstyle:parameternumber") + OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException; + /** * Reads a key from an existing bucket. * @param volumeName Name of the Volume diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 641a63a28f93..f2440a9623f7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1369,36 +1369,11 @@ public OzoneOutputStream createKey( String volumeName, String bucketName, String keyName, long size, ReplicationConfig replicationConfig, Map metadata, Map tags) throws IOException { - createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); - - if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) { - if (tags != null && !tags.isEmpty()) { - throw new IOException("OzoneManager does not support object tags"); - } - } - String ownerName = getRealUserInfo().getShortUserName(); - - OmKeyArgs.Builder builder = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(size) - .setReplicationConfig(replicationConfig) - .addAllMetadataGdpr(metadata) - .addAllTags(tags) - .setLatestVersionLocation(getLatestVersionLocation) - .setOwnerName(ownerName); - - OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); - // For bucket with layout OBJECT_STORE, when create an empty file (size=0), - // OM will set DataSize to OzoneConfigKeys#OZONE_SCM_BLOCK_SIZE, - // which will cause S3G's atomic write length check to fail, - // so reset size to 0 here. - if (isS3GRequest.get() && size == 0) { - openKey.getKeyInfo().setDataSize(size); - } - return createOutputStream(openKey); + OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName, + bucketName, keyName, size, replicationConfig, metadata, tags); + builder.setOwnerName(ownerName); + return openOutputStream(builder.build(), size); } @Override @@ -1411,20 +1386,59 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String Preconditions.checkArgument(existingKeyGeneration > 0, "existingKeyGeneration must be positive, but was %s", existingKeyGeneration); + OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName, + bucketName, keyName, size, replicationConfig, metadata, + Collections.emptyMap()); + builder.setExpectedDataGeneration(existingKeyGeneration); + return openOutputStream(builder.build(), size); + } - createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); + @Override + public OzoneOutputStream createKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { + throw new IOException( + "OzoneManager does not support atomic key creation."); + } + OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName, + bucketName, keyName, size, replicationConfig, metadata, tags); + builder.setExpectedDataGeneration( + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + return openOutputStream(builder.build(), size); + } - OmKeyArgs.Builder builder = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(size) - .setReplicationConfig(replicationConfig) - .addAllMetadataGdpr(metadata) - .setLatestVersionLocation(getLatestVersionLocation) - .setExpectedDataGeneration(existingKeyGeneration); + @Override + @SuppressWarnings("checkstyle:parameternumber") + public OzoneOutputStream rewriteKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { + throw new IOException( + "OzoneManager does not support conditional key rewrite."); + } + OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName, + bucketName, keyName, size, replicationConfig, metadata, tags); + builder.setExpectedETag(expectedETag); + return openOutputStream(builder.build(), size); + } - OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); + private OmKeyArgs.Builder createWriteKeyArgsBuilder(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) + throws IOException { + createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); + validateObjectTagsSupport(tags); + return new OmKeyArgs.Builder(volumeName, bucketName, keyName, size, + replicationConfig, metadata, tags, getLatestVersionLocation); + } + + private OzoneOutputStream openOutputStream(OmKeyArgs keyArgs, long size) + throws IOException { + OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs); // For bucket with layout OBJECT_STORE, when create an empty file (size=0), // OM will set DataSize to OzoneConfigKeys#OZONE_SCM_BLOCK_SIZE, // which will cause S3G's atomic write length check to fail, @@ -1435,6 +1449,15 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String return createOutputStream(openKey); } + private void validateObjectTagsSupport(Map tags) + throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) { + if (tags != null && !tags.isEmpty()) { + throw new IOException("OzoneManager does not support object tags"); + } + } + } + private void createKeyPreChecks(String volumeName, String bucketName, String keyName, ReplicationConfig replicationConfig) throws IOException { verifyVolumeName(volumeName); @@ -1474,33 +1497,61 @@ public OzoneDataStreamOutput createStreamKey( String volumeName, String bucketName, String keyName, long size, ReplicationConfig replicationConfig, Map metadata, Map tags) throws IOException { - verifyVolumeName(volumeName); - verifyBucketName(bucketName); - if (checkKeyNameEnabled) { - HddsClientUtils.verifyKeyName(keyName); - } - HddsClientUtils.checkNotNull(keyName); + OmKeyArgs.Builder builder = createStreamKeyArgsBuilder( + volumeName, bucketName, keyName, size, replicationConfig, metadata, + tags); + return openDataStreamOutput(builder.build()); + } - if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) { - if (tags != null && !tags.isEmpty()) { - throw new IOException("OzoneManager does not support object tags"); - } + @Override + public OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { + throw new IOException( + "OzoneManager does not support atomic key creation."); } + OmKeyArgs.Builder builder = createStreamKeyArgsBuilder( + volumeName, bucketName, keyName, size, replicationConfig, metadata, + tags); + builder.setExpectedDataGeneration( + OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS); + return openDataStreamOutput(builder.build()); + } - String ownerName = getRealUserInfo().getShortUserName(); + @Override + @SuppressWarnings("checkstyle:parameternumber") + public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) { + throw new IOException( + "OzoneManager does not support conditional key rewrite."); + } + OmKeyArgs.Builder builder = createStreamKeyArgsBuilder( + volumeName, bucketName, keyName, size, replicationConfig, metadata, + tags); + builder.setExpectedETag(expectedETag); + return openDataStreamOutput(builder.build()); + } - OmKeyArgs.Builder builder = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(size) - .setReplicationConfig(replicationConfig) - .addAllMetadataGdpr(metadata) - .addAllTags(tags) - .setSortDatanodesInPipeline(true) - .setOwnerName(ownerName); + private OmKeyArgs.Builder createStreamKeyArgsBuilder(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) + throws IOException { + OmKeyArgs.Builder builder = createWriteKeyArgsBuilder( + volumeName, bucketName, keyName, size, replicationConfig, metadata, + tags); + builder.setOwnerName(getRealUserInfo().getShortUserName()); + return builder; + } - OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); + private OzoneDataStreamOutput openDataStreamOutput(OmKeyArgs keyArgs) + throws IOException { + OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs); return createDataStreamOutput(openKey); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 2b5b5559f92b..dc9d16188c57 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -275,5 +275,9 @@ public enum ResultCodes { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD, TOO_MANY_SNAPSHOTS, + + ETAG_MISMATCH, + + ETAG_NOT_AVAILABLE, } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java index dfe0329fbe67..5d2de09c48e5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java @@ -61,6 +61,7 @@ public final class OmKeyArgs extends WithMetadata implements Auditable { // This allows a key to be created an committed atomically if the original has not // been modified. private Long expectedDataGeneration = null; + private final String expectedETag; private OmKeyArgs(Builder b) { super(b); @@ -82,6 +83,7 @@ private OmKeyArgs(Builder b) { this.ownerName = b.ownerName; this.tags = b.tags.build(); this.expectedDataGeneration = b.expectedDataGeneration; + this.expectedETag = b.expectedETag; } public boolean getIsMultipartKey() { @@ -164,6 +166,10 @@ public Long getExpectedDataGeneration() { return expectedDataGeneration; } + public String getExpectedETag() { + return expectedETag; + } + @Override public Map toAuditMap() { Map auditMap = new LinkedHashMap<>(); @@ -209,6 +215,9 @@ public KeyArgs toProtobuf() { if (expectedDataGeneration != null) { builder.setExpectedDataGeneration(expectedDataGeneration); } + if (expectedETag != null) { + builder.setExpectedETag(expectedETag); + } return builder.build(); } @@ -234,11 +243,28 @@ public static class Builder extends WithMetadata.Builder { private boolean forceUpdateContainerCacheFromSCM; private final MapBuilder tags; private Long expectedDataGeneration = null; + private String expectedETag; public Builder() { this(AclListBuilder.empty()); } + @SuppressWarnings("checkstyle:parameternumber") + public Builder(String volumeName, String bucketName, String keyName, + long dataSize, ReplicationConfig replicationConfig, + Map metadata, Map tags, + boolean latestVersionLocation) { + this(); + setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDataSize(dataSize) + .setReplicationConfig(replicationConfig) + .addAllMetadataGdpr(metadata) + .addAllTags(tags) + .setLatestVersionLocation(latestVersionLocation); + } + private Builder(AclListBuilder acls) { this.acls = acls; this.tags = MapBuilder.empty(); @@ -263,6 +289,7 @@ public Builder(OmKeyArgs obj) { this.forceUpdateContainerCacheFromSCM = obj.forceUpdateContainerCacheFromSCM; this.expectedDataGeneration = obj.expectedDataGeneration; + this.expectedETag = obj.expectedETag; this.tags = MapBuilder.of(obj.tags); this.acls = AclListBuilder.of(obj.acls); } @@ -398,6 +425,11 @@ public Builder setExpectedDataGeneration(long generation) { return this; } + public Builder setExpectedETag(String eTag) { + this.expectedETag = eTag; + return this; + } + public OmKeyArgs build() { return new OmKeyArgs(this); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index b0e26c49d695..e388df27b4b3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -110,6 +110,7 @@ public final class OmKeyInfo extends WithParentObjectId // This allows a key to be created an committed atomically if the original has not // been modified. private Long expectedDataGeneration = null; + private String expectedETag; private OmKeyInfo(Builder b) { super(b); @@ -129,6 +130,7 @@ private OmKeyInfo(Builder b) { this.ownerName = b.ownerName; this.tags = b.tags.build(); this.expectedDataGeneration = b.expectedDataGeneration; + this.expectedETag = b.expectedETag; } private static Codec newCodec(boolean ignorePipeline) { @@ -189,6 +191,14 @@ public Long getExpectedDataGeneration() { return expectedDataGeneration; } + public void setExpectedETag(String eTag) { + this.expectedETag = eTag; + } + + public String getExpectedETag() { + return expectedETag; + } + public String getOwnerName() { return ownerName; } @@ -492,6 +502,7 @@ public static class Builder extends WithParentObjectId.Builder { private boolean isFile; private final MapBuilder tags; private Long expectedDataGeneration = null; + private String expectedETag; public Builder() { this.acls = AclListBuilder.empty(); @@ -514,6 +525,7 @@ public Builder(OmKeyInfo obj) { this.fileChecksum = obj.fileChecksum; this.isFile = obj.isFile; this.expectedDataGeneration = obj.expectedDataGeneration; + this.expectedETag = obj.expectedETag; this.tags = MapBuilder.of(obj.tags); obj.keyLocationVersions.forEach(keyLocationVersion -> this.omKeyLocationInfoGroups.add( @@ -685,6 +697,11 @@ public Builder setExpectedDataGeneration(Long existingGeneration) { return this; } + public Builder setExpectedETag(String eTag) { + this.expectedETag = eTag; + return this; + } + @Override protected void validate() { super.validate(); @@ -804,6 +821,9 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, if (expectedDataGeneration != null) { kb.setExpectedDataGeneration(expectedDataGeneration); } + if (expectedETag != null) { + kb.setExpectedETag(expectedETag); + } if (ownerName != null) { kb.setOwnerName(ownerName); } @@ -857,6 +877,9 @@ public static Builder builderFromProtobuf(KeyInfo keyInfo) { if (keyInfo.hasExpectedDataGeneration()) { builder.setExpectedDataGeneration(keyInfo.getExpectedDataGeneration()); } + if (keyInfo.hasExpectedETag()) { + builder.setExpectedETag(keyInfo.getExpectedETag()); + } if (keyInfo.hasOwnerName()) { builder.setOwnerName(keyInfo.getOwnerName()); @@ -997,6 +1020,21 @@ public boolean hasBlocks() { return false; } + public boolean hasEtag() { + return getMetadata().containsKey(OzoneConsts.ETAG); + } + + public boolean isEtagEquals(String matchingETag) { + String currentETag = getMetadata().get(OzoneConsts.ETAG); + if (currentETag == null) { + return false; + } + if (matchingETag.equals("*")) { + return true; + } + return currentETag.equals(matchingETag); + } + public static boolean isKeyEmpty(@Nullable OmKeyInfo keyInfo) { if (keyInfo == null) { return true; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 20f0babab82b..74c88edf0703 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -744,6 +744,9 @@ public OpenKeySession openKey(OmKeyArgs args) throws IOException { if (args.getExpectedDataGeneration() != null) { keyArgs.setExpectedDataGeneration(args.getExpectedDataGeneration()); } + if (args.getExpectedETag() != null) { + keyArgs.setExpectedETag(args.getExpectedETag()); + } req.setKeyArgs(keyArgs.build()); diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot new file mode 100644 index 000000000000..93bd25a43e2b --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation S3 Conditional Put (If-None-Match / If-Match) tests +Library OperatingSystem +Library String +Library Process +Resource ../commonlib.robot +Resource ./commonawslib.robot +Test Timeout 5 minutes +Suite Setup Setup s3 tests + +*** Variables *** +${ENDPOINT_URL} http://s3g:9878 +${BUCKET} generated + +*** Test Cases *** + +Conditional Put If-None-Match Star Creates New Key + [Documentation] If-None-Match: * should succeed when key does not exist + ${key} = Set Variable condput-ifnonematch-new + Execute echo "test-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match "*" + Should contain ${result} ETag + +Conditional Put If-None-Match Star Fails For Existing Key + [Documentation] If-None-Match: * should fail with 412 when key already exists + ${key} = Set Variable condput-ifnonematch-existing + Execute echo "initial-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} + Should contain ${result} ETag + # Now try again with If-None-Match: * + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match "*" + Should contain ${result} PreconditionFailed + +Conditional Put If-Match With Correct ETag Succeeds + [Documentation] If-Match with correct ETag should succeed + ${key} = Set Variable condput-ifmatch-success + Execute echo "initial-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} + Should contain ${result} ETag + # Extract the ETag value + ${etag} = Evaluate __import__('json').loads(r'''${result}''')['ETag'].strip('"') + # Rewrite with matching ETag + Execute echo "updated-content" > /tmp/${key}-updated + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}-updated --if-match ${etag} + Should contain ${result} ETag + ${new_etag} = Evaluate __import__('json').loads(r'''${result}''')['ETag'].strip('"') + Should Not Be Equal ${new_etag} ${etag} + ${head_result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${key} + ${head_etag} = Evaluate __import__('json').loads(r'''${head_result}''')['ETag'].strip('"') + Should Be Equal ${head_etag} ${new_etag} + +Conditional Put If-Match With Wrong ETag Fails + [Documentation] If-Match with wrong ETag should fail with 412 + ${key} = Set Variable condput-ifmatch-fail + Execute echo "initial-content" > /tmp/${key} + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} + Should contain ${result} ETag + ${etag} = Evaluate __import__('json').loads(r'''${result}''')['ETag'].strip('"') + # Try to rewrite with a wrong ETag + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match wrong-etag + Should contain ${result} PreconditionFailed + ${head_result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${key} + ${head_etag} = Evaluate __import__('json').loads(r'''${head_result}''')['ETag'].strip('"') + Should Be Equal ${head_etag} ${etag} + +Conditional Put If-Match On Non-Existent Key Fails + [Documentation] If-Match on a key that does not exist should fail with 412 + ${key} = Set Variable condput-ifmatch-nonexistent + Execute echo "test-content" > /tmp/${key} + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match some-etag + Should contain ${result} PreconditionFailed + ${head_result} = Execute AWSS3APICli and ignore error head-object --bucket ${BUCKET} --key ${key} + Should contain ${head_result} 404 + Should contain ${head_result} Not Found diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 7fb2f885725e..d8fbd1f1ce39 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -25,6 +25,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -377,6 +378,117 @@ public void testPutObject() { assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); } + @Test + public void testPutObjectIfNoneMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + PutObjectRequest request = new PutObjectRequest( + bucketName, keyName, is, new ObjectMetadata()).ifNoneMatch("*"); + + PutObjectResult putObjectResult = s3Client.putObject(request); + assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); + } + + @Test + public void testPutObjectIfNoneMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + + InputStream is2 = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + PutObjectRequest request = new PutObjectRequest( + bucketName, keyName, is2, new ObjectMetadata()).ifNoneMatch("*"); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.putObject(request)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(412, ase.getStatusCode()); + assertEquals("PreconditionFailed", ase.getErrorCode()); + } + + @Test + public void testPutObjectIfMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + String etag = putObjectResult.getETag(); + + InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); + PutObjectRequest request = new PutObjectRequest( + bucketName, keyName, is2, new ObjectMetadata()).ifMatch(etag); + + PutObjectResult putObjectResult2 = s3Client.putObject(request); + assertNotNull(putObjectResult2.getETag()); + assertNotEquals(etag, putObjectResult2.getETag()); + + ObjectMetadata updatedObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); + assertEquals(putObjectResult2.getETag(), updatedObjectMetadata.getETag()); + } + + @Test + public void testPutObjectIfMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + PutObjectResult initialResult = + s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + + InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8)); + PutObjectRequest request = new PutObjectRequest( + bucketName, keyName, is2, new ObjectMetadata()).ifMatch("wrong-etag"); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.putObject(request)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(412, ase.getStatusCode()); + assertEquals("PreconditionFailed", ase.getErrorCode()); + + ObjectMetadata existingObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName); + assertEquals(initialResult.getETag(), existingObjectMetadata.getETag()); + } + + @Test + public void testPutObjectIfMatchMissingKeyFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(bucketName); + + InputStream is = new ByteArrayInputStream("bar2".getBytes( + StandardCharsets.UTF_8)); + PutObjectRequest request = new PutObjectRequest( + bucketName, keyName, is, new ObjectMetadata()).ifMatch("some-etag"); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.putObject(request)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(412, ase.getStatusCode()); + assertEquals("PreconditionFailed", ase.getErrorCode()); + + AmazonServiceException missingKey = assertThrows(AmazonServiceException.class, + () -> s3Client.getObject(bucketName, keyName)); + assertEquals(ErrorType.Client, missingKey.getErrorType()); + assertEquals(404, missingKey.getStatusCode()); + assertEquals("NoSuchKey", missingKey.getErrorCode()); + } + @Test public void testPutObjectWithMD5Header() throws Exception { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 175dcea265f8..35a53dd328bb 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -236,6 +237,107 @@ public void testPutObject() { assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", getObjectResponse.eTag()); } + @Test + public void testPutObjectIfNoneMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + PutObjectResponse putObjectResponse = s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifNoneMatch("*"), + RequestBody.fromString(content)); + + assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", putObjectResponse.eTag()); + } + + @Test + public void testPutObjectIfNoneMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content)); + + S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifNoneMatch("*"), + RequestBody.fromString(content))); + + assertEquals(412, exception.statusCode()); + assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode()); + } + + @Test + public void testPutObjectIfMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + PutObjectResponse initialResponse = s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + + PutObjectResponse putObjectResponse = s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifMatch(initialResponse.eTag()), + RequestBody.fromString("bar2")); + + assertNotNull(putObjectResponse.eTag()); + assertNotEquals(initialResponse.eTag(), putObjectResponse.eTag()); + + HeadObjectResponse headObjectResponse = s3Client.headObject( + b -> b.bucket(bucketName).key(keyName)); + assertEquals(putObjectResponse.eTag(), headObjectResponse.eTag()); + } + + @Test + public void testPutObjectIfMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + PutObjectResponse initialResponse = s3Client.putObject( + b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content)); + + S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifMatch("wrong-etag"), + RequestBody.fromString("bar2"))); + + assertEquals(412, exception.statusCode()); + assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode()); + + HeadObjectResponse headObjectResponse = s3Client.headObject( + b -> b.bucket(bucketName).key(keyName)); + assertEquals(initialResponse.eTag(), headObjectResponse.eTag()); + } + + @Test + public void testPutObjectIfMatchMissingKeyFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .ifMatch("some-etag"), + RequestBody.fromString("bar2"))); + + assertEquals(412, exception.statusCode()); + assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode()); + assertThrows(NoSuchKeyException.class, () -> s3Client.headObject( + b -> b.bucket(bucketName).key(keyName))); + } + @Test public void testPutObjectEmpty() throws Exception { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 378316c6b15e..ec1d522b721e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -42,6 +42,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME; @@ -1482,6 +1483,125 @@ void cannotRewriteRenamedKey(BucketLayout layout) throws IOException { assertThat(e).hasMessageContaining("not found"); } + @ParameterizedTest + @EnumSource + void testCreateKeyIfNotExistsSuccess(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + String keyName = "create-if-not-exists-" + layout.name(); + byte[] content = "test-content".getBytes(UTF_8); + + try (OzoneOutputStream out = bucket.createKeyIfNotExists( + keyName, content.length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + Collections.emptyMap(), Collections.emptyMap())) { + out.write(content); + } + + assertKeyContent(bucket, keyName, content); + } + + @ParameterizedTest + @EnumSource + void testCreateKeyIfNotExistsFailsWhenKeyExists(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKey(bucket); + byte[] newContent = "new-content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.createKeyIfNotExists( + keyDetails.getName(), newContent.length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + Collections.emptyMap(), Collections.emptyMap())) { + out.write(newContent); + } + }); + assertEquals(KEY_ALREADY_EXISTS, e.getResult()); + } + + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchSuccess(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKeyWithETag(bucket); + String etag = keyDetails.getMetadata().get(ETAG); + assertNotNull(etag, "Key should have an ETag"); + + byte[] newContent = "rewritten-content".getBytes(UTF_8); + Map rewrittenMetadata = new HashMap<>(keyDetails.getMetadata()); + String rewrittenETag = UUID.randomUUID().toString(); + rewrittenMetadata.put(ETAG, rewrittenETag); + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + keyDetails.getName(), newContent.length, etag, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + rewrittenMetadata, Collections.emptyMap())) { + out.write(newContent); + } + + assertKeyContent(bucket, keyDetails.getName(), newContent); + assertEquals(rewrittenETag, bucket.getKey(keyDetails.getName()) + .getMetadata().get(ETAG)); + } + + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchFailsWithWrongETag(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKeyWithETag(bucket); + byte[] newContent = "rewritten-content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + keyDetails.getName(), newContent.length, "wrong-etag", + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + keyDetails.getMetadata(), Collections.emptyMap())) { + out.write(newContent); + } + }); + assertEquals(OMException.ResultCodes.ETAG_MISMATCH, e.getResult()); + } + + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchFailsWhenETagNotAvailable(BucketLayout layout) + throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + OzoneKeyDetails keyDetails = createTestKey(bucket); + byte[] newContent = "rewritten-content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + keyDetails.getName(), newContent.length, "some-etag", + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + keyDetails.getMetadata(), Collections.emptyMap())) { + out.write(newContent); + } + }); + assertEquals(OMException.ResultCodes.ETAG_NOT_AVAILABLE, e.getResult()); + } + + @ParameterizedTest + @EnumSource + void testRewriteKeyIfMatchFailsWhenKeyNotFound(BucketLayout layout) throws IOException { + checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY); + OzoneBucket bucket = createBucket(layout); + byte[] content = "content".getBytes(UTF_8); + + OMException e = assertThrows(OMException.class, () -> { + try (OzoneOutputStream out = bucket.rewriteKeyIfMatch( + "nonexistent-key", content.length, "some-etag", + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + Collections.emptyMap(), Collections.emptyMap())) { + out.write(content); + } + }); + assertEquals(KEY_NOT_FOUND, e.getResult()); + } + private static void rewriteKey( OzoneBucket bucket, OzoneKeyDetails keyDetails, byte[] newContent ) throws IOException { @@ -4483,9 +4603,29 @@ private OzoneKeyDetails createTestKey( private OzoneKeyDetails createTestKey( OzoneBucket bucket, String keyName, byte[] bytes + ) throws IOException { + return createTestKey(bucket, keyName, bytes, createTestKeyMetadata()); + } + + private OzoneKeyDetails createTestKeyWithETag(OzoneBucket bucket) + throws IOException { + Map metadata = createTestKeyMetadata(); + metadata.put(ETAG, UUID.randomUUID().toString()); + return createTestKey(bucket, getTestName(), + UUID.randomUUID().toString().getBytes(UTF_8), metadata); + } + + private static Map createTestKeyMetadata() { + Map metadata = new HashMap<>(); + metadata.put("key", RandomStringUtils.secure().nextAscii(10)); + return metadata; + } + + private OzoneKeyDetails createTestKey( + OzoneBucket bucket, String keyName, byte[] bytes, + Map metadata ) throws IOException { RatisReplicationConfig replication = RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE); - Map metadata = singletonMap("key", RandomStringUtils.secure().nextAscii(10)); try (OzoneOutputStream out = bucket.createKey(keyName, bytes.length, replication, metadata)) { out.write(bytes); } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index b6bed6b01981..0cf4dc1b35d1 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -568,6 +568,10 @@ enum Status { KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97; TOO_MANY_SNAPSHOTS = 98; + + ETAG_MISMATCH = 99; + + ETAG_NOT_AVAILABLE = 100; } /** @@ -1085,6 +1089,11 @@ message KeyArgs { // This allows a key to be created an committed atomically if the original has not // been modified. optional uint64 expectedDataGeneration = 23; + + // expectedETag, when set, indicates that the existing key must have + // the given ETag for the operation to succeed. This is used for + // S3 conditional writes with the If-Match header. + optional string expectedETag = 24; } message KeyLocation { @@ -1176,6 +1185,11 @@ message KeyInfo { // This allows a key to be created an committed atomically if the original has not // been modified. optional uint64 expectedDataGeneration = 22; + + // expectedETag, when set, indicates that the existing key must have + // the given ETag for the operation to succeed. This is used for + // S3 conditional writes with the If-Match header. + optional string expectedETag = 23; } // KeyInfoProtoLight is a lightweight subset of KeyInfo message containing diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 492d698db997..65b3394b5fcd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -309,6 +309,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Set the UpdateID to current transactionLogIndex omKeyInfo = omKeyInfo.toBuilder() .setExpectedDataGeneration(null) + .setExpectedETag(null) .addAllMetadata(KeyValueUtil.getFromProtobuf( commitKeyArgs.getMetadataList())) .setUpdateID(trxnLogIndex) @@ -641,6 +642,24 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map } } } + + if (toCommit.getExpectedETag() != null) { + String expectedETag = toCommit.getExpectedETag(); + auditMap.put("expectedETag", expectedETag); + + if (existing == null) { + throw new OMException("Key not found for If-Match at commit", + OMException.ResultCodes.KEY_NOT_FOUND); + } + if (!existing.hasEtag()) { + throw new OMException("Key does not have an ETag at commit", + OMException.ResultCodes.ETAG_NOT_AVAILABLE); + } + if (!existing.isEtagEquals(expectedETag)) { + throw new OMException("ETag changed during write (concurrent modification)", + OMException.ResultCodes.ETAG_MISMATCH); + } + } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java index 25b5a4b15d41..5aa4fbeb36f4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java @@ -246,6 +246,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Optimistic locking validation has passed. Now set the rewrite fields to null so they are // not persisted in the key table. omKeyInfo.setExpectedDataGeneration(null); + omKeyInfo.setExpectedETag(null); long correctedSpace = omKeyInfo.getReplicatedSize(); // if keyToDelete isn't null, usedNamespace shouldn't check and increase. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 1298ff7426fb..4d37a1d7aa7f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -496,5 +496,21 @@ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) } } } + + if (keyArgs.hasExpectedETag()) { + String expectedETag = keyArgs.getExpectedETag(); + if (dbKeyInfo == null) { + throw new OMException("Key not found for If-Match", + OMException.ResultCodes.KEY_NOT_FOUND); + } + if (!dbKeyInfo.hasEtag()) { + throw new OMException("Key does not have an ETag", + OMException.ResultCodes.ETAG_NOT_AVAILABLE); + } + if (!dbKeyInfo.isEtagEquals(expectedETag)) { + throw new OMException("ETag mismatch", + OMException.ResultCodes.ETAG_MISMATCH); + } + } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 7df2619e9e46..13dc904476dd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -993,6 +993,9 @@ protected OmKeyInfo prepareFileInfo( if (keyArgs.hasExpectedDataGeneration()) { builder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); } + if (keyArgs.hasExpectedETag()) { + builder.setExpectedETag(keyArgs.getExpectedETag()); + } return builder.build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 652a7aa0fcf1..1f40b0f72a00 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -350,6 +350,143 @@ public void testAtomicCreateIfNotExistsCommitKeyAlreadyExists() throws Exception assertEquals(KEY_ALREADY_EXISTS, omClientResponse.getOMResponse().getStatus()); } + @Test + public void testCommitWithExpectedETagSuccess() throws Exception { + Table openKeyTable = + omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = + omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = + doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = + getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = + modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + String expectedETag = "matching-etag"; + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedETag(expectedETag); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, + omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + // Add closed key with matching ETag + OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())) + .addMetadata(OzoneConsts.ETAG, expectedETag).build(); + closedKeyTable.put(getOzonePathKey(), closedKeyInfo); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); + + OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); + assertNotNull(committedKey); + assertNull(committedKey.getExpectedETag()); + } + + @Test + public void testCommitWithExpectedETagMismatch() throws Exception { + Table openKeyTable = + omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = + omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = + doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = + getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = + modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedETag("expected-etag"); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, + omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + // Add closed key with different ETag + OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())) + .addMetadata(OzoneConsts.ETAG, "different-etag").build(); + closedKeyTable.put(getOzonePathKey(), closedKeyInfo); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_MISMATCH, + omClientResponse.getOMResponse().getStatus()); + } + + @Test + public void testCommitWithExpectedETagNoETagOnKey() throws Exception { + Table openKeyTable = + omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = + omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = + doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = + getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = + modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setExpectedETag("expected-etag"); + + String openKey = addKeyToOpenKeyTable(allocatedLocationList, + omKeyInfoBuilder); + assertNotNull(openKeyTable.get(openKey)); + + // Add closed key WITHOUT ETag + OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, + new OmKeyLocationInfoGroup(version, new ArrayList<>())).build(); + closedKeyTable.put(getOzonePathKey(), closedKeyInfo); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_NOT_AVAILABLE, + omClientResponse.getOMResponse().getStatus()); + } + @Test public void testValidateAndUpdateCacheWithUncommittedBlocks() throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index 52c9eeea07dd..f9364d87710d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -230,6 +230,126 @@ public void testCreateKeyExpectedGenMismatchReturnsKeyGenerationMismatch( assertNull(openKeyInfo); } + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagKeyNotFound( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag("some-etag")); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagNoETagOnKey( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag("some-etag")); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + // Create existing key without ETag metadata + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig) + .setUpdateID(1L).build(); + omMetadataManager.getKeyTable(getBucketLayout()) + .put(getOzoneKey(), existingKeyInfo); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_NOT_AVAILABLE, + response.getOMResponse().getStatus()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagMismatch( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag("expected-etag")); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + // Create existing key with a different ETag + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig) + .setUpdateID(1L) + .addMetadata(OzoneConsts.ETAG, "different-etag") + .build(); + omMetadataManager.getKeyTable(getBucketLayout()) + .put(getOzoneKey(), existingKeyInfo); + + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals( + OzoneManagerProtocolProtos.Status.ETAG_MISMATCH, + response.getOMResponse().getStatus()); + } + + @ParameterizedTest + @MethodSource("data") + public void testCreateWithExpectedETagSuccess( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + String expectedETag = "matching-etag"; + OMRequest modifiedOmRequest = doPreExecute( + createKeyRequestWithExpectedETag(expectedETag)); + OMKeyCreateRequest omKeyCreateRequest = + getOMKeyCreateRequest(modifiedOmRequest); + + addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, + getBucketLayout()); + + // Create existing key with matching ETag + OmKeyInfo existingKeyInfo = createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig) + .setUpdateID(1L) + .addMetadata(OzoneConsts.ETAG, expectedETag) + .build(); + omMetadataManager.getKeyTable(getBucketLayout()) + .put(getOzoneKey(), existingKeyInfo); + + long id = modifiedOmRequest.getCreateKeyRequest().getClientID(); + OMClientResponse response = + omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, response.getOMResponse().getStatus()); + + // Verify open key was created with expectedETag + OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()) + .get(getOpenKey(id)); + assertNotNull(openKeyInfo); + assertEquals(expectedETag, openKeyInfo.getExpectedETag()); + // Creation time should remain the same on rewrite + assertEquals(existingKeyInfo.getCreationTime(), + openKeyInfo.getCreationTime()); + } + @ParameterizedTest @MethodSource("data") public void testValidateAndUpdateCache( @@ -909,6 +1029,30 @@ private OMRequest createKeyRequest( .setCreateKeyRequest(createKeyRequest).build(); } + private OMRequest createKeyRequestWithExpectedETag(String expectedETag) { + KeyArgs.Builder keyArgs = KeyArgs.newBuilder() + .setVolumeName(volumeName).setBucketName(bucketName) + .setKeyName(keyName).setIsMultipartKey(false) + .setFactor( + ((RatisReplicationConfig) replicationConfig) + .getReplicationFactor()) + .setType(replicationConfig.getReplicationType()) + .setLatestVersionLocation(true) + .setDataSize(100L); + + if (expectedETag != null) { + keyArgs.setExpectedETag(expectedETag); + } + + CreateKeyRequest createKeyRequest = + CreateKeyRequest.newBuilder().setKeyArgs(keyArgs).build(); + + return OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.CreateKey) + .setClientId(UUID.randomUUID().toString()) + .setCreateKeyRequest(createKeyRequest).build(); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) public void testKeyCreateWithFileSystemPathsEnabled( diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index b18cf35d0d32..ad67ebd25908 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -187,8 +187,18 @@ public Response put( throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex); } else if (ex.getResult() == ResultCodes.FILE_ALREADY_EXISTS) { throw newError(S3ErrorTable.NO_OVERWRITE, keyPath, ex); + } else if (ex.getResult() == ResultCodes.KEY_ALREADY_EXISTS) { + throw newError(PRECOND_FAILED, keyPath, ex); + } else if (ex.getResult() == ResultCodes.ETAG_MISMATCH) { + throw newError(PRECOND_FAILED, keyPath, ex); + } else if (ex.getResult() == ResultCodes.ETAG_NOT_AVAILABLE) { + throw newError(PRECOND_FAILED, keyPath, ex); } else if (ex.getResult() == ResultCodes.INVALID_REQUEST) { throw newError(S3ErrorTable.INVALID_REQUEST, keyPath); + } else if (ex.getResult() == ResultCodes.KEY_NOT_FOUND + && getHeaders().getHeaderString(S3Consts.IF_MATCH_HEADER) != null) { + // If-Match failed because the key doesn't exist + throw newError(PRECOND_FAILED, keyPath, ex); } else if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) { throw newError(S3ErrorTable.NO_SUCH_KEY, keyPath); } else if (ex.getResult() == ResultCodes.NOT_SUPPORTED_OPERATION) { @@ -230,7 +240,6 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr copyHeader = getHeaders().getHeaderString(COPY_SOURCE_HEADER); - // Normal put object ReplicationConfig replicationConfig = getReplicationConfig(bucket); boolean enableEC = false; @@ -271,6 +280,38 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr return Response.ok().status(HttpStatus.SC_OK).build(); } + String ifNoneMatch = getHeaders().getHeaderString( + S3Consts.IF_NONE_MATCH_HEADER); + String ifMatch = getHeaders().getHeaderString( + S3Consts.IF_MATCH_HEADER); + + if (ifNoneMatch != null && StringUtils.isBlank(ifNoneMatch)) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage("If-None-Match header cannot be empty."); + throw ex; + } + if (ifMatch != null && StringUtils.isBlank(ifMatch)) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage("If-Match header cannot be empty."); + throw ex; + } + + String ifNoneMatchTrimmed = ifNoneMatch == null ? null : ifNoneMatch.trim(); + String ifMatchTrimmed = ifMatch == null ? null : ifMatch.trim(); + + if (ifNoneMatchTrimmed != null && ifMatchTrimmed != null) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage("If-Match and If-None-Match cannot be specified together."); + throw ex; + } + + if (ifNoneMatchTrimmed != null + && !"*".equals(stripQuotes(ifNoneMatchTrimmed))) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage("Only If-None-Match: * is supported for conditional put."); + throw ex; + } + // Normal put object S3ChunkInputStreamInfo chunkInputStreamInfo = getS3ChunkInputStreamInfo(body, length, amzDecodedLength, keyPath); @@ -287,15 +328,18 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr perf.appendStreamMode(); Pair keyWriteResult = ObjectEndpointStreaming .put(bucket, keyPath, length, replicationConfig, getChunkSize(), - customMetadata, tags, multiDigestInputStream, getHeaders(), signatureInfo.isSignPayload(), perf); + customMetadata, tags, multiDigestInputStream, getHeaders(), + signatureInfo.isSignPayload(), perf, ifNoneMatchTrimmed, + ifMatchTrimmed); md5Hash = keyWriteResult.getKey(); putLength = keyWriteResult.getValue(); } else { final String amzContentSha256Header = validateSignatureHeader(getHeaders(), keyPath, signatureInfo.isSignPayload()); - try (OzoneOutputStream output = getClientProtocol().createKey( - volume.getName(), bucketName, keyPath, length, replicationConfig, - customMetadata, tags)) { + try (OzoneOutputStream output = openKeyForPut( + volume.getName(), bucketName, keyPath, length, + replicationConfig, customMetadata, tags, ifNoneMatchTrimmed, + ifMatchTrimmed)) { long metadataLatencyNs = getMetrics().updatePutKeyMetadataStats(startNanos); perf.appendMetaLatencyNanos(metadataLatencyNs); @@ -1127,7 +1171,43 @@ private CopyObjectResponse copyObject(OzoneVolume volume, } } } - + + /** + * Opens a key for put, applying conditional write logic based on + * If-None-Match and If-Match headers. + */ + @SuppressWarnings("checkstyle:ParameterNumber") + private OzoneOutputStream openKeyForPut(String volumeName, String bucketName, String keyPath, long length, + ReplicationConfig replicationConfig, Map customMetadata, + Map tags, String ifNoneMatch, String ifMatch) + throws IOException { + if (ifNoneMatch != null && "*".equals(stripQuotes(ifNoneMatch.trim()))) { + return getClientProtocol().createKeyIfNotExists( + volumeName, bucketName, keyPath, length, replicationConfig, + customMetadata, tags); + } else if (ifMatch != null) { + String expectedETag = parseETag(ifMatch); + return getClientProtocol().rewriteKeyIfMatch( + volumeName, bucketName, keyPath, length, expectedETag, + replicationConfig, customMetadata, tags); + } else { + return getClientProtocol().createKey( + volumeName, bucketName, keyPath, length, replicationConfig, + customMetadata, tags); + } + } + + /** + * Parses an ETag from a conditional header value, removing surrounding + * quotes if present. + */ + static String parseETag(String headerValue) { + if (headerValue == null) { + return null; + } + return stripQuotes(headerValue.trim()); + } + /** Request context shared among {@code ObjectOperationHandler}s. */ final class ObjectRequestContext extends S3RequestContext { private final String bucketName; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java index 767c11506dc1..2eb2dd21f30c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -73,12 +73,13 @@ public static Pair put( int chunkSize, Map keyMetadata, Map tags, MultiDigestInputStream body, HttpHeaders headers, boolean isSignedPayload, - PerformanceStringBuilder perf) + PerformanceStringBuilder perf, String ifNoneMatch, String ifMatch) throws IOException, OS3Exception { try { return putKeyWithStream(bucket, keyPath, - length, chunkSize, replicationConfig, keyMetadata, tags, body, headers, isSignedPayload, perf); + length, chunkSize, replicationConfig, keyMetadata, tags, body, + headers, isSignedPayload, perf, ifNoneMatch, ifMatch); } catch (IOException ex) { LOG.error("Exception occurred in PutObject", ex); if (ex instanceof OMException) { @@ -113,14 +114,17 @@ public static Pair putKeyWithStream( MultiDigestInputStream body, HttpHeaders headers, boolean isSignedPayload, - PerformanceStringBuilder perf) + PerformanceStringBuilder perf, + String ifNoneMatch, + String ifMatch) throws IOException, OS3Exception { long startNanos = Time.monotonicNowNanos(); final String amzContentSha256Header = validateSignatureHeader(headers, keyPath, isSignedPayload); long writeLen; String md5Hash; - try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath, - length, replicationConfig, keyMetadata, tags)) { + try (OzoneDataStreamOutput streamOutput = openStreamKeyForPut(bucket, + keyPath, length, replicationConfig, keyMetadata, tags, ifNoneMatch, + ifMatch)) { long metadataLatencyNs = METRICS.updatePutKeyMetadataStats(startNanos); writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length); md5Hash = DatatypeConverter.printHexBinary(body.getMessageDigest(OzoneConsts.MD5_HASH).digest()) @@ -155,6 +159,24 @@ public static Pair putKeyWithStream( return Pair.of(md5Hash, writeLen); } + @SuppressWarnings("checkstyle:ParameterNumber") + private static OzoneDataStreamOutput openStreamKeyForPut(OzoneBucket bucket, + String keyPath, long length, ReplicationConfig replicationConfig, + Map keyMetadata, Map tags, + String ifNoneMatch, String ifMatch) throws IOException { + if (ifNoneMatch != null && "*".equals(ObjectEndpoint.parseETag(ifNoneMatch))) { + return bucket.createStreamKeyIfNotExists(keyPath, length, + replicationConfig, keyMetadata, tags); + } + if (ifMatch != null) { + return bucket.rewriteStreamKeyIfMatch(keyPath, length, + ObjectEndpoint.parseETag(ifMatch), replicationConfig, keyMetadata, + tags); + } + return bucket.createStreamKey(keyPath, length, replicationConfig, + keyMetadata, tags); + } + @SuppressWarnings("checkstyle:ParameterNumber") public static long copyKeyWithStream( OzoneBucket bucket, diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index 797ca1f36712..e90612cc56ed 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -98,6 +98,10 @@ public final class S3Consts { public static final String CHECKSUM_HEADER = "Content-MD5"; + // Conditional request headers + public static final String IF_NONE_MATCH_HEADER = "If-None-Match"; + public static final String IF_MATCH_HEADER = "If-Match"; + //Never Constructed private S3Consts() { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 739babce1d06..32a969fddbc0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -249,6 +249,45 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String .rewriteKey(keyName, size, existingKeyGeneration, replicationConfig, metadata); } + @Override + public OzoneOutputStream createKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return getBucket(volumeName, bucketName) + .createKeyIfNotExists(keyName, size, replicationConfig, metadata, tags); + } + + @Override + public OzoneOutputStream rewriteKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return getBucket(volumeName, bucketName) + .rewriteKeyIfMatch(keyName, size, expectedETag, replicationConfig, + metadata, tags); + } + + @Override + public OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName, + String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return getBucket(volumeName, bucketName) + .createStreamKeyIfNotExists(keyName, size, replicationConfig, + metadata, tags); + } + + @Override + public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName, + String bucketName, String keyName, long size, String expectedETag, + ReplicationConfig replicationConfig, Map metadata, + Map tags) throws IOException { + return getBucket(volumeName, bucketName) + .rewriteStreamKeyIfMatch(keyName, size, expectedETag, + replicationConfig, metadata, tags); + } + @Override public OzoneInputStream getKey(String volumeName, String bucketName, String keyName) throws IOException { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 8037f65fda1c..75cc73aadf20 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -104,7 +104,7 @@ public OzoneBucketStub build() { return new OzoneBucketStub(this); } } - + @Override public OzoneOutputStream createKey(String key, long size) throws IOException { return createKey(key, size, @@ -192,6 +192,38 @@ public void close() throws IOException { return new OzoneOutputStream(byteArrayOutputStream, null); } + @Override + public OzoneOutputStream createKeyIfNotExists(String keyName, long size, + ReplicationConfig rConfig, Map metadata, + Map tags) throws IOException { + if (keyDetails.containsKey(keyName)) { + throw new OMException("Key already exists", + ResultCodes.KEY_ALREADY_EXISTS); + } + return createKey(keyName, size, rConfig, metadata, tags); + } + + @Override + public OzoneOutputStream rewriteKeyIfMatch(String keyName, long size, + String expectedETag, ReplicationConfig rConfig, + Map metadata, Map tags) + throws IOException { + OzoneKeyDetails existing = keyDetails.get(keyName); + if (existing == null) { + throw new OMException("Key not found for If-Match", + ResultCodes.KEY_NOT_FOUND); + } + if (!existing.hasEtag()) { + throw new OMException("Key does not have an ETag", + ResultCodes.ETAG_NOT_AVAILABLE); + } + if (!existing.isEtagEquals(expectedETag)) { + throw new OMException("ETag mismatch", + ResultCodes.ETAG_MISMATCH); + } + return createKey(keyName, size, rConfig, metadata, tags); + } + @Override public OzoneDataStreamOutput createStreamKey(String key, long size, ReplicationConfig rConfig, @@ -247,6 +279,38 @@ public void flush() throws IOException { return new OzoneDataStreamOutputStub(byteBufferStreamOutput, key + size); } + @Override + public OzoneDataStreamOutput createStreamKeyIfNotExists(String key, long size, + ReplicationConfig rConfig, Map keyMetadata, + Map tags) throws IOException { + if (keyDetails.containsKey(key)) { + throw new OMException("Key already exists", + ResultCodes.KEY_ALREADY_EXISTS); + } + return createStreamKey(key, size, rConfig, keyMetadata, tags); + } + + @Override + public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String key, long size, + String expectedETag, ReplicationConfig rConfig, + Map keyMetadata, Map tags) + throws IOException { + OzoneKeyDetails existing = keyDetails.get(key); + if (existing == null) { + throw new OMException("Key not found for If-Match", + ResultCodes.KEY_NOT_FOUND); + } + if (!existing.hasEtag()) { + throw new OMException("Key does not have an ETag", + ResultCodes.ETAG_NOT_AVAILABLE); + } + if (!existing.isEtagEquals(expectedETag)) { + throw new OMException("ETag mismatch", + ResultCodes.ETAG_MISMATCH); + } + return createStreamKey(key, size, rConfig, keyMetadata, tags); + } + @Override public OzoneDataStreamOutput createMultipartStreamKey(String key, long size, diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index c2456dd068fd..e85b121aaaf5 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -555,6 +555,113 @@ private HttpHeaders newMockHttpHeaders() { return httpHeaders; } + @Test + void testIfNoneMatchKeyDoesNotExistSuccess() throws Exception { + when(headers.getHeaderString("If-None-Match")).thenReturn("*"); + + assertSucceeds(() -> putObject(CONTENT)); + assertKeyContent(bucket, KEY_NAME, CONTENT); + } + + @Test + void testIfNoneMatchKeyExistsPreconditionFailed() throws Exception { + // First create the key + assertSucceeds(() -> putObject(CONTENT)); + + // Now try to create again with If-None-Match: * + when(headers.getHeaderString("If-None-Match")).thenReturn("*"); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.PRECOND_FAILED, () -> putObject(CONTENT)); + assertNotNull(ex); + } + + @Test + void testIfMatchETagMatchesSuccess() throws Exception { + // First create the key to get an ETag + Response response = putObject(CONTENT); + String etag = response.getHeaderString(HttpHeaders.ETAG); + assertNotNull(etag); + + // Now try to rewrite with matching ETag + when(headers.getHeaderString("If-Match")).thenReturn(etag); + + assertSucceeds(() -> putObject("new-content")); + assertKeyContent(bucket, KEY_NAME, "new-content"); + } + + @Test + void testIfMatchETagMismatchPreconditionFailed() throws Exception { + // First create the key + assertSucceeds(() -> putObject(CONTENT)); + + // Try to rewrite with wrong ETag + when(headers.getHeaderString("If-Match")).thenReturn("\"wrong-etag\""); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.PRECOND_FAILED, () -> putObject("new-content")); + assertNotNull(ex); + } + + @Test + void testIfMatchKeyNotFoundPreconditionFailed() throws Exception { + // Try If-Match on a non-existent key + when(headers.getHeaderString("If-Match")).thenReturn("\"some-etag\""); + + OS3Exception ex = assertErrorResponse( + S3ErrorTable.PRECOND_FAILED, () -> putObject(CONTENT)); + assertNotNull(ex); + } + + @Test + void testBothHeadersProvidedInvalidRequest() throws Exception { + when(headers.getHeaderString("If-None-Match")).thenReturn("*"); + when(headers.getHeaderString("If-Match")).thenReturn("\"some-etag\""); + + OS3Exception ex = assertErrorResponse( + INVALID_REQUEST, () -> putObject(CONTENT)); + assertNotNull(ex); + assertThat(ex.getErrorMessage()).contains( + "If-Match and If-None-Match cannot be specified together"); + } + + @Test + void testBlankIfNoneMatchInvalidRequest() throws Exception { + when(headers.getHeaderString("If-None-Match")).thenReturn(" "); + + OS3Exception ex = assertErrorResponse( + INVALID_REQUEST, () -> putObject(CONTENT)); + assertThat(ex.getErrorMessage()).contains( + "If-None-Match header cannot be empty"); + } + + @Test + void testBlankIfMatchInvalidRequest() throws Exception { + when(headers.getHeaderString("If-Match")).thenReturn(" "); + + OS3Exception ex = assertErrorResponse( + INVALID_REQUEST, () -> putObject(CONTENT)); + assertThat(ex.getErrorMessage()).contains("If-Match header cannot be empty"); + } + + @Test + void testIfNoneMatchNotStarInvalidRequest() throws Exception { + when(headers.getHeaderString("If-None-Match")).thenReturn("\"etag\""); + + OS3Exception ex = assertErrorResponse( + INVALID_REQUEST, () -> putObject(CONTENT)); + assertThat(ex.getErrorMessage()).contains( + "Only If-None-Match: * is supported"); + } + + @Test + void testParseETag() { + assertEquals("abc123", ObjectEndpoint.parseETag("\"abc123\"")); + assertEquals("abc123", ObjectEndpoint.parseETag("abc123")); + assertEquals("abc123", ObjectEndpoint.parseETag(" \"abc123\" ")); + assertEquals(null, ObjectEndpoint.parseETag(null)); + } + /** Put object at {@code bucketName}/{@code keyName} with pre-defined {@link #CONTENT}. */ private Response putObject(String bucketName, String keyName) throws IOException, OS3Exception { return put(objectEndpoint, bucketName, keyName, CONTENT);