diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 7e98b1279c03..b4d319f85d41 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -281,6 +281,7 @@ public static boolean isReadOnly(OMRequest omRequest) { case GetObjectTagging: case GetQuotaRepairStatus: case StartQuotaRepair: + case FinalizeUpgrade: return true; case CreateVolume: case SetVolumeProperty: @@ -314,7 +315,6 @@ public static boolean isReadOnly(OMRequest omRequest) { // As protobuf currently doesn't support deprecating enum fields // TODO: Remove once migrated to proto3 and mark fields in proto // as deprecated - case FinalizeUpgrade: case Prepare: case CancelPrepare: case DeleteOpenKeys: @@ -396,6 +396,7 @@ public static boolean shouldSendToFollower(OMRequest omRequest) { case GetKeyInfo: case GetSnapshotInfo: case GetObjectTagging: + case FinalizeUpgrade: return true; case CreateVolume: case SetVolumeProperty: @@ -429,7 +430,6 @@ public static boolean shouldSendToFollower(OMRequest omRequest) { // As protobuf currently doesn't support deprecating enum fields // TODO: Remove once migrated to proto3 and mark fields in proto // as deprecated - case FinalizeUpgrade: case Prepare: case CancelPrepare: case DeleteOpenKeys: diff --git a/hadoop-ozone/dist/src/main/smoketest/upgrade/check-finalization.robot b/hadoop-ozone/dist/src/main/smoketest/upgrade/check-finalization.robot index 717a2998f794..476f36b463d0 100644 --- a/hadoop-ozone/dist/src/main/smoketest/upgrade/check-finalization.robot +++ b/hadoop-ozone/dist/src/main/smoketest/upgrade/check-finalization.robot @@ -24,6 +24,7 @@ Test Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit t *** Test Cases *** Check OM Finalized [Tags] om finalized + Skip OM cannot finalize now/yet ${result} = Execute env Log ${result} Pass Execution If '%{OZONE_UPGRADE_FROM}' == '1.1.0' OM finalization not supported in version %{OZONE_UPGRADE_FROM} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java deleted file mode 100644 index 0c4f9c97b7bc..000000000000 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.ozone; - -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; -import static org.apache.hadoop.ozone.OzoneConsts.OZONE_ROOT; -import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_LEASE_HARD_LIMIT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting; -import static org.apache.ozone.test.LambdaTestUtils.await; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.conf.StorageUnit; -import org.apache.hadoop.hdds.scm.storage.BlockInputStream; -import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; -import org.apache.hadoop.hdds.scm.storage.BufferPool; -import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigForTesting; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; -import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; -import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl; -import org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore; -import org.apache.hadoop.ozone.om.OMStorage; -import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; -import org.apache.ozone.test.GenericTestUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; -import org.slf4j.event.Level; - -/** - * Test HSync upgrade. - */ -@TestMethodOrder(OrderAnnotation.class) -public class TestHSyncUpgrade { - private MiniOzoneCluster cluster; - private OzoneBucket bucket; - - private final OzoneConfiguration conf = new OzoneConfiguration(); - private OzoneClient client; - private static final BucketLayout BUCKET_LAYOUT = BucketLayout.FILE_SYSTEM_OPTIMIZED; - - private static final int CHUNK_SIZE = 4 << 12; - private static final int FLUSH_SIZE = 3 * CHUNK_SIZE; - private static final int MAX_FLUSH_SIZE = 2 * FLUSH_SIZE; - private static final int BLOCK_SIZE = 2 * MAX_FLUSH_SIZE; - private static final int SERVICE_INTERVAL = 100; - private static final int EXPIRE_THRESHOLD_MS = 140; - - private static final int POLL_INTERVAL_MILLIS = 500; - private static final int POLL_MAX_WAIT_MILLIS = 120_000; - - @BeforeEach - public void init() throws Exception { - final BucketLayout layout = BUCKET_LAYOUT; - - conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); - conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true); - conf.setBoolean("ozone.client.hbase.enhancements.allowed", true); - conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); - conf.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10); - // Reduce KeyDeletingService interval - conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); - conf.setTimeDuration(OZONE_DIR_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); - conf.setBoolean("ozone.client.incremental.chunk.list", true); - conf.setBoolean("ozone.client.stream.putblock.piggybacking", true); - conf.setTimeDuration(OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL, - SERVICE_INTERVAL, TimeUnit.MILLISECONDS); - conf.setTimeDuration(OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD, - EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS); - conf.setTimeDuration(OZONE_OM_LEASE_HARD_LIMIT, - EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS); - conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s"); - conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY, OMLayoutFeature.MULTITENANCY_SCHEMA.layoutVersion()); - - ClientConfigForTesting.newBuilder(StorageUnit.BYTES) - .setBlockSize(BLOCK_SIZE) - .setChunkSize(CHUNK_SIZE) - .setStreamBufferFlushSize(FLUSH_SIZE) - .setStreamBufferMaxSize(MAX_FLUSH_SIZE) - .setDataStreamBufferFlushSize(MAX_FLUSH_SIZE) - .setDataStreamMinPacketSize(CHUNK_SIZE) - .setDataStreamWindowSize(5 * CHUNK_SIZE) - .applyTo(conf); - - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(5) - .build(); - cluster.waitForClusterToBeReady(); - client = cluster.newClient(); - - // create a volume and a bucket to be used by OzoneFileSystem - bucket = TestDataUtil.createVolumeAndBucket(client, layout); - - // Enable DEBUG level logging for relevant classes - GenericTestUtils.setLogLevel(BlockManagerImpl.class, Level.DEBUG); - GenericTestUtils.setLogLevel(AbstractDatanodeStore.class, Level.DEBUG); - GenericTestUtils.setLogLevel(BlockOutputStream.class, Level.DEBUG); - GenericTestUtils.setLogLevel(BlockInputStream.class, Level.DEBUG); - GenericTestUtils.setLogLevel(KeyValueHandler.class, Level.DEBUG); - - GenericTestUtils.setLogLevel(BufferPool.class, Level.DEBUG); - - OpenKeyCleanupService openKeyCleanupService = - (OpenKeyCleanupService) cluster.getOzoneManager().getKeyManager() - .getOpenKeyCleanupService(); - openKeyCleanupService.suspend(); - } - - @AfterEach - public void teardown() { - IOUtils.closeQuietly(client); - if (cluster != null) { - cluster.shutdown(); - } - } - - @Test - public void upgrade() throws Exception { - preFinalizationChecks(); - finalizeOMUpgrade(); - } - - private void preFinalizationChecks() throws IOException { - final String rootPath = String.format("%s://%s/", - OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY)); - conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); - - final String dir = OZONE_ROOT + bucket.getVolumeName() - + OZONE_URI_DELIMITER + bucket.getName(); - - final Path file = new Path(dir, "pre-finalization"); - try (RootedOzoneFileSystem fs = (RootedOzoneFileSystem)FileSystem.get(conf)) { - try (FSDataOutputStream outputStream = fs.create(file, true)) { - OMException omException = assertThrows(OMException.class, outputStream::hsync); - assertFinalizationExceptionForHsync(omException); - } - final OzoneManagerProtocol omClient = client.getObjectStore() - .getClientProxy().getOzoneManagerClient(); - OMException omException = assertThrows(OMException.class, - () -> omClient.listOpenFiles("", 100, "")); - assertFinalizationException(omException); - - omException = assertThrows(OMException.class, - () -> fs.recoverLease(file)); - assertFinalizationException(omException); - - fs.delete(file, false); - } - } - - private void assertFinalizationExceptionForHsync(OMException omException) { - assertEquals(NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION, - omException.getResult()); - assertThat(omException.getMessage()) - .contains("Cluster does not have the hsync support feature finalized yet"); - } - - private void assertFinalizationException(OMException omException) { - assertEquals(NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION, - omException.getResult()); - assertThat(omException.getMessage()) - .contains("cannot be invoked before finalization."); - } - - /** - * Trigger OM upgrade finalization from the client and block until completion - * (status FINALIZATION_DONE). - */ - private void finalizeOMUpgrade() throws Exception { - // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call - final OzoneManagerProtocol omClient = client.getObjectStore() - .getClientProxy().getOzoneManagerClient(); - final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID(); - UpgradeFinalization.StatusAndMessages finalizationResponse = - omClient.finalizeUpgrade(upgradeClientID); - - // The status should transition as soon as the client call above returns - assertTrue(isStarting(finalizationResponse.status())); - // Wait for the finalization to be marked as done. - // 10s timeout should be plenty. - await(POLL_MAX_WAIT_MILLIS, POLL_INTERVAL_MILLIS, () -> { - final UpgradeFinalization.StatusAndMessages progress = - omClient.queryUpgradeFinalizationProgress( - upgradeClientID, false, false); - return isDone(progress.status()); - }); - } - -} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java index 58b9bdc21725..88cc4ea1db5d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java @@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; import org.apache.ozone.test.LambdaTestUtils; +import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; @@ -70,6 +71,7 @@ */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@Unhealthy("OM cannot finalize now/yet") class TestOMBucketLayoutUpgrade { private static final int PRE_UPGRADE = 100; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index be0f99099770..dd08e79a7978 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine; import org.apache.hadoop.ozone.upgrade.UpgradeFinalization.StatusAndMessages; +import org.apache.ozone.test.tag.Unhealthy; import org.apache.ratis.util.LifeCycle; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; @@ -50,6 +51,7 @@ * Tests for OM upgrade finalization. * TODO: can be merged into class with other OM tests with per-method cluster */ +@Unhealthy("OM cannot finalize now until it can poll the SCM for the trigger") class TestOMUpgradeFinalization { static { AuditLogTestUtils.enableAuditLog(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java index 82ac9855ad09..5f757595f0b4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java @@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.LambdaTestUtils.VoidCallable; +import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -60,6 +61,7 @@ * and that users not belonging to a tenant are directed to the default S3 * volume. */ +@Unhealthy("OM cannot finalize now/yet") public class TestMultiTenantVolume { private static MiniOzoneCluster cluster; private static String s3VolumeName; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 5f14451fad32..1001ba73aaac 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -161,6 +161,7 @@ import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.tag.Slow; +import org.apache.ozone.test.tag.Unhealthy; import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; @@ -171,6 +172,7 @@ * Abstract class to test OmSnapshot. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Unhealthy("no trigger for finalization now/yet, all implementations try finalizing") public abstract class TestOmSnapshot { static { Logger.getLogger(ManagedRocksObjectUtils.class).setLevel(Level.DEBUG); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java index e6185f3d65a0..a775d2f96b01 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java @@ -91,7 +91,6 @@ private static void init() { CMD_AUDIT_ACTION_MAP.put(Type.SetBucketProperty, OMAction.UPDATE_BUCKET); CMD_AUDIT_ACTION_MAP.put(Type.Prepare, OMAction.UPGRADE_PREPARE); CMD_AUDIT_ACTION_MAP.put(Type.CancelPrepare, OMAction.UPGRADE_CANCEL); - CMD_AUDIT_ACTION_MAP.put(Type.FinalizeUpgrade, OMAction.UPGRADE_FINALIZE); CMD_AUDIT_ACTION_MAP.put(Type.GetObjectTagging, OMAction.GET_OBJECT_TAGGING); CMD_AUDIT_ACTION_MAP.put(Type.PutObjectTagging, OMAction.PUT_OBJECT_TAGGING); CMD_AUDIT_ACTION_MAP.put(Type.DeleteObjectTagging, OMAction.DELETE_OBJECT_TAGGING); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 1778de3520d9..a124e85bb808 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -85,7 +85,6 @@ import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotRenameRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotSetPropertyRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMCancelPrepareRequest; -import org.apache.hadoop.ozone.om.request.upgrade.OMFinalizeUpgradeRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMPrepareRequest; import org.apache.hadoop.ozone.om.request.util.OMEchoRPCWriteRequest; import org.apache.hadoop.ozone.om.request.volume.OMQuotaRepairRequest; @@ -183,8 +182,6 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMRenewDelegationTokenRequest(omRequest); case GetS3Secret: return new S3GetSecretRequest(omRequest); - case FinalizeUpgrade: - return new OMFinalizeUpgradeRequest(omRequest); case Prepare: return new OMPrepareRequest(omRequest); case CancelPrepare: diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java deleted file mode 100644 index b37d1ee6d1dc..000000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.request.upgrade; - -import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY; -import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.FinalizeUpgrade; - -import java.io.IOException; -import java.util.HashMap; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus; -import org.apache.hadoop.hdds.utils.db.cache.CacheKey; -import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.audit.AuditLogger; -import org.apache.hadoop.ozone.audit.OMAction; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; -import org.apache.hadoop.ozone.om.request.OMClientRequest; -import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.om.response.upgrade.OMFinalizeUpgradeResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization.StatusAndMessages; -import org.apache.hadoop.security.UserGroupInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Handles finalizeUpgrade request. - */ -public class OMFinalizeUpgradeRequest extends OMClientRequest { - private static final Logger LOG = - LoggerFactory.getLogger(OMFinalizeUpgradeRequest.class); - - public OMFinalizeUpgradeRequest(OMRequest omRequest) { - super(omRequest); - } - - @Override - public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, ExecutionContext context) { - LOG.trace("Request: {}", getOmRequest()); - AuditLogger auditLogger = ozoneManager.getAuditLogger(); - OzoneManagerProtocolProtos.UserInfo userInfo = getOmRequest().getUserInfo(); - OMResponse.Builder responseBuilder = - OmResponseUtil.getOMResponseBuilder(getOmRequest()); - responseBuilder.setCmdType(FinalizeUpgrade); - OMClientResponse response = null; - Exception exception = null; - - try { - if (ozoneManager.getAclsEnabled()) { - UserGroupInformation ugi = createUGIForApi(); - if (!ozoneManager.isAdmin(ugi)) { - throw new OMException("Access denied for user " + ugi + ". " - + "Superuser privilege is required to finalize upgrade.", - OMException.ResultCodes.ACCESS_DENIED); - } - } - - FinalizeUpgradeRequest request = - getOmRequest().getFinalizeUpgradeRequest(); - - String upgradeClientID = request.getUpgradeClientId(); - - StatusAndMessages omStatus = - ozoneManager.finalizeUpgrade(upgradeClientID); - - UpgradeFinalizationStatus.Status protoStatus = - UpgradeFinalizationStatus.Status.valueOf(omStatus.status().name()); - UpgradeFinalizationStatus responseStatus = - UpgradeFinalizationStatus.newBuilder() - .setStatus(protoStatus) - .build(); - - OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); - int lV = ozoneManager.getVersionManager().getMetadataLayoutVersion(); - omMetadataManager.getMetaTable().addCacheEntry( - new CacheKey<>(LAYOUT_VERSION_KEY), - CacheValue.get(context.getIndex(), String.valueOf(lV))); - - FinalizeUpgradeResponse omResponse = - FinalizeUpgradeResponse.newBuilder() - .setStatus(responseStatus) - .build(); - responseBuilder.setFinalizeUpgradeResponse(omResponse); - response = new OMFinalizeUpgradeResponse(responseBuilder.build(), - ozoneManager.getVersionManager().getMetadataLayoutVersion()); - LOG.trace("Returning response: {}", response); - } catch (IOException e) { - exception = e; - response = new OMFinalizeUpgradeResponse( - createErrorOMResponse(responseBuilder, e), -1); - } - - markForAudit(auditLogger, buildAuditMessage(OMAction.UPGRADE_FINALIZE, - new HashMap<>(), exception, userInfo)); - return response; - } - -} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java deleted file mode 100644 index 49bb6fc86587..000000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.response.upgrade; - -import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY; -import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.META_TABLE; - -import java.io.IOException; -import org.apache.hadoop.hdds.utils.db.BatchOperation; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.response.CleanupTableInfo; -import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Response for finalizeUpgrade request. - */ -@CleanupTableInfo(cleanupTables = {META_TABLE}) -public class OMFinalizeUpgradeResponse extends OMClientResponse { - private static final Logger LOG = - LoggerFactory.getLogger(OMFinalizeUpgradeResponse.class); - private int layoutVersionToWrite = -1; - - public OMFinalizeUpgradeResponse( - OzoneManagerProtocolProtos.OMResponse omResponse, - int layoutVersionToWrite) { - super(omResponse); - this.layoutVersionToWrite = layoutVersionToWrite; - } - - @Override - protected void addToDBBatch(OMMetadataManager omMetadataManager, - BatchOperation batchOperation) throws IOException { - if (layoutVersionToWrite != -1) { - LOG.info("Layout version to persist to DB : {}", layoutVersionToWrite); - omMetadataManager.getMetaTable().putWithBatch(batchOperation, - LAYOUT_VERSION_KEY, - String.valueOf(layoutVersionToWrite)); - } - } -} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index bea7785bfbc2..941a54460c11 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.protocolPB; +import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser; import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.FILESYSTEM_SNAPSHOT; import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.HBASE_SUPPORT; import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.MULTITENANCY_SCHEMA; @@ -108,6 +109,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.EchoRPCResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeProgressRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeProgressResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetKeyInfoRequest; @@ -166,6 +168,7 @@ import org.apache.hadoop.ozone.upgrade.UpgradeFinalization.StatusAndMessages; import org.apache.hadoop.ozone.util.PayloadUtils; import org.apache.hadoop.ozone.util.ProtobufUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -391,6 +394,10 @@ public OMResponse handleReadRequest(OMRequest request) { getObjectTagging(request.getGetObjectTaggingRequest()); responseBuilder.setGetObjectTaggingResponse(getObjectTaggingResponse); break; + case FinalizeUpgrade: + FinalizeUpgradeResponse finalizeUpgradeResponse = finalizeUpgrade(); + responseBuilder.setFinalizeUpgradeResponse(finalizeUpgradeResponse); + break; default: responseBuilder.setSuccess(false); responseBuilder.setMessage("Unrecognized Command Type: " + cmdType); @@ -1549,6 +1556,35 @@ private GetObjectTaggingResponse getObjectTagging(GetObjectTaggingRequest reques return resp.build(); } + private FinalizeUpgradeResponse finalizeUpgrade() + throws IOException { + FinalizeUpgradeResponse response = null; + + UserGroupInformation ugi = getRemoteUser(); + if (!impl.isAdmin(ugi)) { + throw new OMException("Access denied for user " + ugi + ". " + + "Superuser privilege is required to finalize upgrade.", + OMException.ResultCodes.ACCESS_DENIED); + } + + UpgradeFinalizationStatus.Status protoStatus = + UpgradeFinalizationStatus.Status.ALREADY_FINALIZED; + + UpgradeFinalizationStatus responseStatus = + UpgradeFinalizationStatus.newBuilder() + .setStatus(protoStatus) + .build(); + + response = + FinalizeUpgradeResponse.newBuilder() + .setStatus(responseStatus) + .build(); + + LOG.trace("Returning response: {}", response); + return response; + + } + private SafeModeAction toSafeModeAction( OzoneManagerProtocolProtos.SafeMode safeMode) { switch (safeMode) {