diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index 892dd4de1ff8..1dd75081388b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -450,6 +450,8 @@ StatusAndMessages queryUpgradeFinalizationProgress( String upgradeClientID, boolean force, boolean readonly) throws IOException; + HddsProtos.UpgradeStatus queryUpgradeStatus(String upgradeClientID, boolean readonly) throws IOException; + DecommissionScmResponseProto decommissionScm( String scmId) throws IOException; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 92ddfa7eb8dc..6cf8274e6b3b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -484,6 +484,8 @@ StatusAndMessages queryUpgradeFinalizationProgress( String upgradeClientID, boolean force, boolean readonly) throws IOException; + HddsProtos.UpgradeStatus queryUpgradeStatus(String upgradeClientID, boolean readonly) throws IOException; + /** * Obtain a token which can be used to let datanodes verify authentication of * commands operating on {@code containerID}. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index b079936baa10..8b1674c9968f 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -1151,6 +1151,21 @@ public StatusAndMessages queryUpgradeFinalizationProgress( status.getMessagesList()); } + @Override + public HddsProtos.UpgradeStatus queryUpgradeStatus(String upgradeClientID, boolean readonly) throws IOException { + StorageContainerLocationProtocolProtos.QueryUpgradeStatusRequestProto req = + StorageContainerLocationProtocolProtos.QueryUpgradeStatusRequestProto + .newBuilder() + .setReadonly(readonly) + .setUpgradeClientId(upgradeClientID) + .build(); + + StorageContainerLocationProtocolProtos.QueryUpgradeStatusResponseProto response = + submitRequest(Type.QueryUpgradeStatus, builder -> builder.setQueryUpgradeStatusRequest(req)) + .getQueryUpgradeStatusResponse(); + return response.getStatus(); + } + @Override public Token getContainerToken( ContainerID containerID) throws IOException { diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto index f80a50a3be97..522433c8a022 100644 --- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto +++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto @@ -87,6 +87,7 @@ message ScmContainerLocationRequest { optional ContainerBalancerStatusInfoRequestProto containerBalancerStatusInfoRequest = 48; optional ReconcileContainerRequestProto reconcileContainerRequest = 49; optional GetDeletedBlocksTxnSummaryRequestProto getDeletedBlocksTxnSummaryRequest = 50; + optional QueryUpgradeStatusRequestProto queryUpgradeStatusRequest = 51; } message ScmContainerLocationResponse { @@ -145,6 +146,7 @@ message ScmContainerLocationResponse { optional ContainerBalancerStatusInfoResponseProto containerBalancerStatusInfoResponse = 48; optional ReconcileContainerResponseProto reconcileContainerResponse = 49; optional GetDeletedBlocksTxnSummaryResponseProto getDeletedBlocksTxnSummaryResponse = 50; + optional QueryUpgradeStatusResponseProto queryUpgradeStatusResponse = 51; enum Status { OK = 1; @@ -202,6 +204,7 @@ enum Type { GetContainerBalancerStatusInfo = 44; ReconcileContainer = 45; GetDeletedBlocksTransactionSummary = 46; + QueryUpgradeStatus = 47; } /** @@ -570,6 +573,15 @@ message QueryUpgradeFinalizationProgressResponseProto { required hadoop.hdds.UpgradeFinalizationStatus status = 1; } +message QueryUpgradeStatusRequestProto { + required string upgradeClientId = 1; + optional bool readonly = 2; +} + +message QueryUpgradeStatusResponseProto { + required hadoop.hdds.UpgradeStatus status = 1; +} + message ContainerTokenSecretProto { required string ownerId = 1; required ContainerID containerId = 2; diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto index fd908b38e6b0..6217b03c6574 100644 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@ -426,6 +426,13 @@ message UpgradeFinalizationStatus { repeated string messages = 2; } +message UpgradeStatus { + optional bool scmFinalized = 1; + optional int32 numDatanodesFinalized = 2; + optional int32 numDatanodesTotal = 3; + optional bool shouldFinalize = 4; +} + /** * Information for Certificate Revocation List. */ diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 176de420c2cd..50f17a21a695 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -748,6 +748,12 @@ public ScmContainerLocationResponse processRequest( .setStatus(Status.OK) .setReconcileContainerResponse(reconcileContainer(request.getReconcileContainerRequest())) .build(); + case QueryUpgradeStatus: + return ScmContainerLocationResponse.newBuilder() + .setCmdType(request.getCmdType()) + .setStatus(Status.OK) + .setQueryUpgradeStatusResponse(getQueryUpgradeStatus(request.getQueryUpgradeStatusRequest())) + .build(); default: throw new IllegalArgumentException( "Unknown command type: " + request.getCmdType()); @@ -1088,6 +1094,16 @@ public FinalizeScmUpgradeResponseProto getFinalizeScmUpgrade( .build(); } + public StorageContainerLocationProtocolProtos.QueryUpgradeStatusResponseProto getQueryUpgradeStatus( + StorageContainerLocationProtocolProtos.QueryUpgradeStatusRequestProto request) throws IOException { + + HddsProtos.UpgradeStatus response = impl.queryUpgradeStatus(request.getUpgradeClientId(), request.getReadonly()); + return StorageContainerLocationProtocolProtos.QueryUpgradeStatusResponseProto + .newBuilder() + .setStatus(response) + .build(); + } + public ForceExitSafeModeResponseProto forceExitSafeMode( ForceExitSafeModeRequestProto request) throws IOException { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 107496152747..5b62f9880fd9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -1148,6 +1148,32 @@ public StatusAndMessages queryUpgradeFinalizationProgress( } } + @Override + public HddsProtos.UpgradeStatus queryUpgradeStatus(String upgradeClientID, boolean readonly) throws IOException { + Map auditMap = Maps.newHashMap(); + auditMap.put("upgradeClientID", upgradeClientID); + auditMap.put("readonly", String.valueOf(readonly)); + + try { + if (!readonly) { + getScm().checkAdminAccess(getRemoteUser(), true); + } + + // Returning a placeholder for now. + HddsProtos.UpgradeStatus result = HddsProtos.UpgradeStatus.newBuilder() + .setScmFinalized(true) + .setNumDatanodesFinalized(10) + .setNumDatanodesTotal(10) + .setShouldFinalize(true) + .build(); + AUDIT.logReadSuccess(buildAuditMessageForSuccess(SCMAction.QUERY_UPGRADE_STATUS, auditMap)); + return result; + } catch (IOException ex) { + AUDIT.logReadFailure(buildAuditMessageForFailure(SCMAction.QUERY_UPGRADE_STATUS, auditMap, ex)); + throw ex; + } + } + @Override public StartContainerBalancerResponseProto startContainerBalancer( Optional threshold, Optional iterations, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java index 52cd943c4dbb..35ffd179cadc 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java @@ -60,6 +60,7 @@ public enum SCMAction implements AuditAction { GET_SAFE_MODE_RULE_STATUSES, FINALIZE_SCM_UPGRADE, QUERY_UPGRADE_FINALIZATION_PROGRESS, + QUERY_UPGRADE_STATUS, GET_DATANODE_USAGE_INFO, GET_CONTAINER_TOKEN, GET_CONTAINER_COUNT, diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index 6e1df27b27b7..7d8bfc3c3e4f 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -582,6 +582,11 @@ public StatusAndMessages queryUpgradeFinalizationProgress( upgradeClientID, force, readonly); } + @Override + public HddsProtos.UpgradeStatus queryUpgradeStatus(String upgradeClientID, boolean readonly) throws IOException { + return storageContainerLocationClient.queryUpgradeStatus(upgradeClientID, readonly); + } + @Override public DecommissionScmResponseProto decommissionScm( String scmId) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java new file mode 100644 index 000000000000..1c741f8fdd2b --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.admin.upgrade; + +import java.io.IOException; +import java.util.UUID; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import picocli.CommandLine; + +/** + * Sub command to query the overall upgrade status of the cluster, returning information about the finalization + * status of SCM, the datanodes and OM. + */ +@CommandLine.Command( + name = "status", + description = "Show status of the cluster upgrade", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class StatusSubCommand extends ScmSubcommand { + + @Override + public void execute(ScmClient client) throws IOException { + String upgradeClientID = "Upgrade-Client-" + UUID.randomUUID(); + HddsProtos.UpgradeStatus status = client.queryUpgradeStatus(upgradeClientID, true); + + // Temporary output to validate the command is working. + out().println("Update status:"); + out().println(" SCM Finalized: " + status.getScmFinalized()); + out().println(" Datanodes finalized: " + status.getNumDatanodesFinalized()); + out().println(" Total Datanodes: " + status.getNumDatanodesTotal()); + out().println(" Should Finalize: " + status.getShouldFinalize()); + } +} diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/UpgradeCommands.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/UpgradeCommands.java new file mode 100644 index 000000000000..a1372e16951b --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/UpgradeCommands.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.admin.upgrade; + +import org.apache.hadoop.hdds.cli.AdminSubcommand; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.kohsuke.MetaInfServices; +import picocli.CommandLine; + +/** + * Subcommand to group upgrade related operations. + */ +@CommandLine.Command( + name = "upgrade", + description = "Operations related to Ozone cluster upgrade", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class, + subcommands = { + StatusSubCommand.class + }) +@MetaInfServices(AdminSubcommand.class) +public class UpgradeCommands implements AdminSubcommand { +} diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/package-info.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/package-info.java new file mode 100644 index 000000000000..608cfb772b3b --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Ozone upgrade cli tools. + */ +package org.apache.hadoop.ozone.admin.upgrade;