From 4ef621351096145381eceeadef784f605bdf7dd6 Mon Sep 17 00:00:00 2001 From: Anurag Parvatikar Date: Fri, 11 Sep 2026 18:34:41 +0530 Subject: [PATCH] HDDS-15881. Support setting storagePolicy via SetBucketProperty --- .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../shell/bucket/CreateBucketHandler.java | 43 +++++++++ .../shell/bucket/UpdateBucketHandler.java | 74 ++++++++++++++++ .../hadoop/ozone/client/OzoneBucket.java | 12 +++ .../ozone/client/protocol/ClientProtocol.java | 11 +++ .../hadoop/ozone/client/rpc/RpcClient.java | 8 ++ .../hadoop/ozone/om/helpers/OmBucketArgs.java | 30 +++++++ .../hadoop/ozone/shell/TestOzoneShellHA.java | 88 +++++++++++++++++++ .../src/main/proto/OmClientProtocol.proto | 1 + .../bucket/OMBucketSetPropertyRequest.java | 11 +++ .../TestOMBucketSetPropertyRequest.java | 73 +++++++++++++++ .../ozone/client/ClientProtocolStub.java | 6 ++ 12 files changed, 358 insertions(+) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 50559ca917ae..1f6f44ecaefb 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -300,6 +300,7 @@ public final class OzoneConsts { public static final String STORAGE_TYPE = "storageType"; public static final String STORAGE_POLICY = "storagePolicy"; public static final String ALLOW_FALLBACK_STORAGE_POLICY = "allowFallbackStoragePolicy"; + public static final String UNSET_STORAGE_POLICY = "unSetStoragePolicy"; public static final String RESOURCE_TYPE = "resourceType"; public static final String IS_VERSION_ENABLED = "isVersionEnabled"; public static final String CREATION_TIME = "creationTime"; diff --git a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java index ab842f184e5c..4576c4f8a211 100644 --- a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java +++ b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java @@ -21,6 +21,8 @@ import java.io.IOException; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.OzoneQuota; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -67,6 +69,19 @@ public class CreateBucketHandler extends BucketHandler { @CommandLine.Mixin private SetSpaceQuotaOptions quotaOptions; + @Option(names = {"--storage-policy", "-s"}, + description = "Bucket StoragePolicy. Allowed values: HOT, WARM, COLD, null. Default: WARM.", + defaultValue = "WARM") + private String storagePolicyStr; + + @Option(names = {"--allow-fallback-storage-policy", "-a"}, + description = "When true, allocation may fall back to the StoragePolicy's " + + "fallback tier if the creation tier is unavailable. Default: true.", + defaultValue = "true") + private String allowFallBackStoragePolicyStr; + + private static final String NULL_STORAGE_POLICY = "null"; + /** * Executes create bucket. */ @@ -78,8 +93,14 @@ public void execute(OzoneClient client, OzoneAddress address) ownerName = UserGroupInformation.getCurrentUser().getShortUserName(); } + StoragePolicy storagePolicy = parseStoragePolicy(storagePolicyStr); + Boolean allowFallBackStoragePolicy = + Boolean.valueOf(allowFallBackStoragePolicyStr); + BucketArgs.Builder bb = new BucketArgs.Builder() + .setStoragePolicy(storagePolicy) + .setAllowFallbackStoragePolicy(allowFallBackStoragePolicy) .setVersioning(false).setOwner(ownerName); if (allowedBucketLayout != null) { bb.setBucketLayout(allowedBucketLayout); @@ -127,6 +148,28 @@ public void execute(OzoneClient client, OzoneAddress address) } } + /** + * Parse a user-supplied {@code --storage-policy} value into a + * {@link StoragePolicy}. Returns {@code null} when the caller passed + * "null" (any case) or an empty value, to leave the bucket without an + * explicit StoragePolicy (the server defaults it to WARM on create). + * + * @throws IllegalArgumentException if the value is not one of HOT, WARM, + * COLD, or "null". + */ + private static StoragePolicy parseStoragePolicy(String value) { + if (Strings.isNullOrEmpty(value) + || NULL_STORAGE_POLICY.equalsIgnoreCase(value)) { + return null; + } + try { + return OzoneStoragePolicy.valueOf(value.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid storage policy: " + value + + ". Allowed String values are: HOT, WARM, COLD, or null."); + } + } + private static class LayoutConverter implements CommandLine.ITypeConverter { @Override public BucketLayout convert(String value) { diff --git a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/UpdateBucketHandler.java b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/UpdateBucketHandler.java index 58f7190628f5..f756779ad49a 100644 --- a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/UpdateBucketHandler.java +++ b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/UpdateBucketHandler.java @@ -17,9 +17,13 @@ package org.apache.hadoop.ozone.shell.bucket; +import com.google.common.base.Strings; import java.io.IOException; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.om.helpers.OmBucketArgs; import org.apache.hadoop.ozone.shell.OzoneAddress; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -35,6 +39,20 @@ public class UpdateBucketHandler extends BucketHandler { description = "Owner of the bucket to set") private String ownerName; + @Option(names = {"--storage-policy", "-s"}, + description = "Bucket StoragePolicy. Allowed values: HOT, WARM, COLD, null " + + "(null clears the bucket's StoragePolicy). Leave unset to keep the " + + "current value.") + private String storagePolicyStr; + + @Option(names = {"--allow-fallback-storage-policy", "-a"}, + description = "When true, allocation may fall back to the StoragePolicy's " + + "fallback tier if the creation tier is unavailable. Leave unset to " + + "keep the current value.") + private String allowFallBackStoragePolicyStr; + + private static final String NULL_STORAGE_POLICY = "null"; + @Override protected void execute(OzoneClient client, OzoneAddress address) throws IOException { @@ -52,8 +70,64 @@ protected void execute(OzoneClient client, OzoneAddress address) } } + // Update StoragePolicy / allowFallback if requested. + boolean shouldSetStoragePolicyProperty = false; + OmBucketArgs.Builder bucketArgsBuilder = OmBucketArgs.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName); + + if (hasStoragePolicy()) { + shouldSetStoragePolicyProperty = true; + StoragePolicy storagePolicy = getStoragePolicy(); + if (storagePolicy == null) { + bucketArgsBuilder.setUnSetStoragePolicy(true); + } else { + bucketArgsBuilder.setStoragePolicy(storagePolicy); + } + } + + Boolean allowFallBackStoragePolicy = getAllowFallBackStoragePolicy(); + if (allowFallBackStoragePolicy != null) { + shouldSetStoragePolicyProperty = true; + bucketArgsBuilder.setAllowFallbackStoragePolicy(allowFallBackStoragePolicy); + } + if (shouldSetStoragePolicyProperty) { + bucket.setStoragePolicyProperty(bucketArgsBuilder.build()); + } + OzoneBucket updatedBucket = client.getObjectStore().getVolume(volumeName) .getBucket(bucketName); printObjectAsJson(updatedBucket); } + + private boolean hasStoragePolicy() { + return !Strings.isNullOrEmpty(storagePolicyStr); + } + + /** + * Parse the {@code --storagepolicy} value. Returns {@code null} when the + * user passed "null" (any case), meaning the policy should be cleared; + * otherwise the matching {@link StoragePolicy}. + * + * @throws IllegalArgumentException if the value is not HOT, WARM, COLD, or + * "null". + */ + private StoragePolicy getStoragePolicy() { + if (Strings.isNullOrEmpty(storagePolicyStr) + || NULL_STORAGE_POLICY.equalsIgnoreCase(storagePolicyStr)) { + return null; + } + try { + return OzoneStoragePolicy.valueOf(storagePolicyStr.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid storage policy: " + + storagePolicyStr + + ". Allowed String values are: HOT, WARM, COLD, or null."); + } + } + + private Boolean getAllowFallBackStoragePolicy() { + return allowFallBackStoragePolicyStr == null ? null + : Boolean.valueOf(allowFallBackStoragePolicyStr); + } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 1999c8144ca7..2c52c515ccf4 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -55,6 +55,7 @@ import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; +import org.apache.hadoop.ozone.om.helpers.OmBucketArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; @@ -377,6 +378,17 @@ public void setStoragePolicy(StoragePolicy newStoragePolicy) throws IOException storagePolicy = newStoragePolicy; } + /** + * Sets the bucket's storage-policy properties carried in the given + * {@link OmBucketArgs} (storage policy, allowFallback, or unset). Used by the + * update path, where the policy may be absent or explicitly cleared. + * @param args Bucket arguments carrying the properties to update. + * @throws IOException + */ + public void setStoragePolicyProperty(OmBucketArgs args) throws IOException { + proxy.setBucketStoragePolicy(args); + } + /** * Enable/Disable versioning of the bucket. * @param newVersioning diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 49d5b82cf3ec..04015000ce1c 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -50,6 +50,7 @@ import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmBucketArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -293,6 +294,16 @@ void setBucketStoragePolicy(String volumeName, String bucketName, StoragePolicy storagePolicy) throws IOException; + /** + * Sets the storage-policy related properties of a bucket carried in the + * given {@link OmBucketArgs} (storage policy, allowFallback, or unset). + * Unlike the three-arg overload, the policy may be absent or explicitly + * cleared via {@link OmBucketArgs.Builder#setUnSetStoragePolicy}. + * @param args Bucket arguments carrying the properties to update. + * @throws IOException + */ + void setBucketStoragePolicy(OmBucketArgs args) throws IOException; + /** * Deletes a bucket if it is empty. * @param volumeName Name of the Volume diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 0218522b1d69..46375d190acc 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1236,6 +1236,14 @@ public void setBucketStoragePolicy( ozoneManagerClient.setBucketProperty(builder.build()); } + @Override + public void setBucketStoragePolicy(OmBucketArgs args) throws IOException { + Objects.requireNonNull(args, "args == null"); + verifyVolumeName(args.getVolumeName()); + verifyBucketName(args.getBucketName()); + ozoneManagerClient.setBucketProperty(args); + } + @Override public void setBucketQuota(String volumeName, String bucketName, long quotaInNamespace, long quotaInBytes) throws IOException { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java index c25173e26142..7fc8a42c8719 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java @@ -55,6 +55,11 @@ public final class OmBucketArgs extends WithMetadata implements Auditable { * the flag was not set. */ private final Boolean allowFallbackStoragePolicy; + /** + * Whether to clear (unset) the bucket's storage policy. {@code null} or + * {@code false} leaves the existing policy unchanged. + */ + private final Boolean unSetStoragePolicy; /** * Bucket encryption key info if encryption is enabled. @@ -81,6 +86,7 @@ private OmBucketArgs(Builder b) { this.isVersionEnabled = b.isVersionEnabled; this.storagePolicy = b.storagePolicy; this.allowFallbackStoragePolicy = b.allowFallbackStoragePolicy; + this.unSetStoragePolicy = b.unSetStoragePolicy; this.ownerName = b.ownerName; this.defaultReplicationConfig = b.defaultReplicationConfig; this.quotaInBytesSet = b.quotaInBytesSet; @@ -132,6 +138,14 @@ public Boolean getAllowFallbackStoragePolicy() { return allowFallbackStoragePolicy; } + /** + * Returns whether the bucket's storage policy should be cleared (unset). + * @return unSetStoragePolicy (may be {@code null} when not set). + */ + public Boolean getUnSetStoragePolicy() { + return unSetStoragePolicy; + } + /** * Returns Bucket Quota in bytes. * @return quotaInBytes. @@ -213,6 +227,10 @@ public Map toAuditMap() { auditMap.put(OzoneConsts.ALLOW_FALLBACK_STORAGE_POLICY, String.valueOf(this.allowFallbackStoragePolicy)); } + if (this.unSetStoragePolicy != null) { + auditMap.put(OzoneConsts.UNSET_STORAGE_POLICY, + String.valueOf(this.unSetStoragePolicy)); + } if (this.ownerName != null) { auditMap.put(OzoneConsts.OWNER, this.ownerName); } @@ -249,6 +267,7 @@ public static class Builder extends WithMetadata.Builder { private Boolean isVersionEnabled; private StoragePolicy storagePolicy; private Boolean allowFallbackStoragePolicy; + private Boolean unSetStoragePolicy; private boolean quotaInBytesSet = false; private long quotaInBytes; private boolean quotaInNamespaceSet = false; @@ -306,6 +325,11 @@ public Builder setAllowFallbackStoragePolicy(Boolean allowFallback) { return this; } + public Builder setUnSetStoragePolicy(Boolean unSet) { + this.unSetStoragePolicy = unSet; + return this; + } + public Builder setQuotaInBytes(long quota) { quotaInBytesSet = true; quotaInBytes = quota; @@ -371,6 +395,9 @@ public BucketArgs getProtobuf() { if (allowFallbackStoragePolicy != null) { builder.setAllowFallbackStoragePolicy(allowFallbackStoragePolicy); } + if (unSetStoragePolicy != null) { + builder.setUnSetStoragePolicy(unSetStoragePolicy); + } if (quotaInBytesSet && ( quotaInBytes > 0 || quotaInBytes == OzoneConsts.QUOTA_RESET)) { builder.setQuotaInBytes(quotaInBytes); @@ -416,6 +443,9 @@ public static Builder builderFromProtobuf(BucketArgs bucketArgs) { if (bucketArgs.hasAllowFallbackStoragePolicy()) { builder.setAllowFallbackStoragePolicy(bucketArgs.getAllowFallbackStoragePolicy()); } + if (bucketArgs.hasUnSetStoragePolicy()) { + builder.setUnSetStoragePolicy(bucketArgs.getUnSetStoragePolicy()); + } if (bucketArgs.hasOwnerName()) { builder.setOwnerName(bucketArgs.getOwnerName()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 00b070e29d21..1d41572650db 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -73,6 +73,7 @@ import org.apache.hadoop.fs.ozone.OzoneTrashPolicy; import org.apache.hadoop.hdds.JsonTestUtils; import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -1278,6 +1279,93 @@ public void testDeleteTrashNoSkipTrash() throws Exception { } + @Test + @SuppressWarnings("methodlength") + public void testShCreateBucketWithStoragePolicy() throws Exception { + ObjectStore objectStore = client.getObjectStore(); + execute(ozoneShell, new String[]{"volume", "create", "spvol1"}); + out.reset(); + + // No -sp: server defaults to WARM and allowFallback true. + execute(ozoneShell, new String[]{"bucket", "create", "spvol1/bucket1"}); + assertThat(objectStore.getVolume("spvol1").getBucket("bucket1") + .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.WARM); + assertThat(objectStore.getVolume("spvol1").getBucket("bucket1") + .getAllowFallbackStoragePolicy()).isTrue(); + + // -sp null on create still yields a non-null policy (defaults to WARM). + execute(ozoneShell, new String[]{"bucket", "create", "spvol1/bucket2", "-s", "null"}); + assertThat(objectStore.getVolume("spvol1").getBucket("bucket2") + .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.WARM); + + execute(ozoneShell, new String[]{"bucket", "create", "spvol1/bucket3", "-s", "HOT"}); + assertThat(objectStore.getVolume("spvol1").getBucket("bucket3") + .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT); + + execute(ozoneShell, + new String[]{"bucket", "create", "spvol1/bucket4", "-s", "HOT", "-a", "false"}); + assertThat(objectStore.getVolume("spvol1").getBucket("bucket4") + .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT); + assertThat(objectStore.getVolume("spvol1").getBucket("bucket4") + .getAllowFallbackStoragePolicy()).isFalse(); + + execute(ozoneShell, new String[]{"bucket", "create", "spvol1/bucket5", "-s", "COLD"}); + assertThat(objectStore.getVolume("spvol1").getBucket("bucket5") + .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.COLD); + + objectStore.getVolume("spvol1").deleteBucket("bucket1"); + objectStore.getVolume("spvol1").deleteBucket("bucket2"); + objectStore.getVolume("spvol1").deleteBucket("bucket3"); + objectStore.getVolume("spvol1").deleteBucket("bucket4"); + objectStore.getVolume("spvol1").deleteBucket("bucket5"); + objectStore.deleteVolume("spvol1"); + } + + @Test + @SuppressWarnings("methodlength") + public void testShUpdateBucketStoragePolicy() throws Exception { + ObjectStore objectStore = client.getObjectStore(); + execute(ozoneShell, new String[]{"volume", "create", "spvol2"}); + out.reset(); + + execute(ozoneShell, new String[]{"bucket", "create", "spvol2/bucket1"}); + assertThat(objectStore.getVolume("spvol2").getBucket("bucket1") + .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.WARM); + assertThat(objectStore.getVolume("spvol2").getBucket("bucket1") + .getAllowFallbackStoragePolicy()).isTrue(); + + // Update policy only; allowFallback is left unchanged. + execute(ozoneShell, new String[]{"bucket", "update", "spvol2/bucket1", "-s", "HOT"}); + assertThat(objectStore.getVolume("spvol2").getBucket("bucket1") + .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT); + assertThat(objectStore.getVolume("spvol2").getBucket("bucket1") + .getAllowFallbackStoragePolicy()).isTrue(); + + // Update both policy and allowFallback. + execute(ozoneShell, + new String[]{"bucket", "update", "spvol2/bucket1", "-s", "COLD", "-a", "false"}); + assertThat(objectStore.getVolume("spvol2").getBucket("bucket1") + .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.COLD); + assertThat(objectStore.getVolume("spvol2").getBucket("bucket1") + .getAllowFallbackStoragePolicy()).isFalse(); + + // Unset the policy via -sp null (case-insensitive). + execute(ozoneShell, new String[]{"bucket", "update", "spvol2/bucket1", "-s", "NULL"}); + assertThat(objectStore.getVolume("spvol2").getBucket("bucket1") + .getStoragePolicy()).isNull(); + + // Set it again, then unset via lowercase null. + execute(ozoneShell, new String[]{"bucket", "update", "spvol2/bucket1", "-s", "HOT"}); + assertThat(objectStore.getVolume("spvol2").getBucket("bucket1") + .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT); + execute(ozoneShell, new String[]{"bucket", "update", "spvol2/bucket1", "-s", "null"}); + assertThat(objectStore.getVolume("spvol2").getBucket("bucket1") + .getStoragePolicy()).isNull(); + + objectStore.getVolume("spvol2").deleteBucket("bucket1"); + objectStore.deleteVolume("spvol2"); + } + @Test @SuppressWarnings("methodlength") public void testShQuota() throws Exception { diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 94b3020f8e36..1b71c90d43a0 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -901,6 +901,7 @@ message BucketArgs { repeated hadoop.hdds.KeyValue tags = 13; optional hadoop.hdds.StoragePolicyProto storagePolicy = 14; optional bool allowFallbackStoragePolicy = 15; + optional bool unSetStoragePolicy = 16; } message PrefixInfo { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java index 2911ba47c530..53e4766a3d9a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java @@ -166,11 +166,22 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut //Check StoragePolicy to update StoragePolicy storagePolicy = omBucketArgs.getStoragePolicy(); + Boolean unSetStoragePolicy = omBucketArgs.getUnSetStoragePolicy(); + if (storagePolicy != null && Boolean.TRUE.equals(unSetStoragePolicy)) { + throw new OMException("Set storagePolicy and unset storagePolicy cannot " + + "be given at the same time", + OMException.ResultCodes.NOT_SUPPORTED_OPERATION); + } if (storagePolicy != null) { bucketInfoBuilder.setStoragePolicy(storagePolicy); LOG.debug("Updating bucket storage policy for bucket: {} in volume: {}", bucketName, volumeName); } + if (Boolean.TRUE.equals(unSetStoragePolicy)) { + bucketInfoBuilder.setStoragePolicy(null); + LOG.debug("Unsetting bucket storage policy for bucket: {} in volume: {}", + bucketName, volumeName); + } //Check allowFallbackStoragePolicy to update Boolean allowFallbackStoragePolicy = diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java index 2e41d4c8b173..fb3611b70b7e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java @@ -29,6 +29,8 @@ import java.util.UUID; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.OzoneStoragePolicy; +import org.apache.hadoop.hdds.client.StoragePolicy; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; @@ -496,4 +498,75 @@ public void testSettingQuotaRetainsReplication() throws Exception { assertEquals(20 * GB, dbBucketInfoAfter.getQuotaInBytes()); assertEquals(1000L, dbBucketInfoAfter.getQuotaInNamespace()); } + + @Test + public void testSettingStoragePolicy() throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + + OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setStoragePolicy(OzoneStoragePolicy.WARM); + OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager); + OMRequestTestUtils.addBucketToDB(omMetadataManager, bucketInfo); + + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + assertThat(omMetadataManager.getBucketTable().get(bucketKey) + .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.WARM); + + // Set the StoragePolicy to HOT and disable fallback. + OMClientResponse response = runSetStoragePolicy(volumeName, bucketName, + OzoneStoragePolicy.HOT, null, false, 1); + assertThat(response.getOMResponse().getSuccess()).isTrue(); + OmBucketInfo updated = omMetadataManager.getBucketTable().get(bucketKey); + assertThat(updated.getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT); + assertThat(updated.getAllowFallbackStoragePolicy()).isFalse(); + + // Set the StoragePolicy to COLD and enable fallback. + response = runSetStoragePolicy(volumeName, bucketName, + OzoneStoragePolicy.COLD, null, true, 2); + assertThat(response.getOMResponse().getSuccess()).isTrue(); + updated = omMetadataManager.getBucketTable().get(bucketKey); + assertThat(updated.getStoragePolicy()).isEqualTo(OzoneStoragePolicy.COLD); + assertThat(updated.getAllowFallbackStoragePolicy()).isTrue(); + + // Unset the StoragePolicy. + response = runSetStoragePolicy(volumeName, bucketName, null, true, null, 3); + assertThat(response.getOMResponse().getSuccess()).isTrue(); + updated = omMetadataManager.getBucketTable().get(bucketKey); + assertThat(updated.getStoragePolicy()).isNull(); + + // Set and unset the StoragePolicy at the same time is not allowed. + response = runSetStoragePolicy(volumeName, bucketName, + OzoneStoragePolicy.HOT, true, null, 4); + assertThat(response.getOMResponse().getSuccess()).isFalse(); + assertThat(response.getOMResponse().getStatus()) + .isEqualTo(OzoneManagerProtocolProtos.Status.NOT_SUPPORTED_OPERATION); + } + + private OMClientResponse runSetStoragePolicy(String volumeName, + String bucketName, StoragePolicy storagePolicy, + Boolean unSetStoragePolicy, Boolean allowFallbackStoragePolicy, + long txnId) throws Exception { + OmBucketArgs.Builder argsBuilder = OmBucketArgs.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName); + if (storagePolicy != null) { + argsBuilder.setStoragePolicy(storagePolicy); + } + if (unSetStoragePolicy != null) { + argsBuilder.setUnSetStoragePolicy(unSetStoragePolicy); + } + if (allowFallbackStoragePolicy != null) { + argsBuilder.setAllowFallbackStoragePolicy(allowFallbackStoragePolicy); + } + OMRequest omRequest = OMRequest.newBuilder().setSetBucketPropertyRequest( + SetBucketPropertyRequest.newBuilder() + .setBucketArgs(argsBuilder.build().getProtobuf())) + .setCmdType(OzoneManagerProtocolProtos.Type.SetBucketProperty) + .setClientId(UUID.randomUUID().toString()).build(); + return new OMBucketSetPropertyRequest(omRequest) + .validateAndUpdateCache(ozoneManager, txnId); + } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 22e85954ae23..a4a516b0cca0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -40,6 +40,7 @@ import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmBucketArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -198,6 +199,11 @@ public void setBucketStoragePolicy(String volumeName, String bucketName, } + @Override + public void setBucketStoragePolicy(OmBucketArgs args) throws IOException { + + } + @Override public void deleteBucket(String volumeName, String bucketName) throws IOException {