* If userGroupInformation is not null, use the passed ugi, else obtain * from {@link UserGroupInformation#getCurrentUser()} + *
+ * Additionally, we can optionally specify {@link SocketFactory} which can be used for use
+ * case when two clients require different timeout configuration. This is because Hadoop
+ * {@link org.apache.hadoop.ipc.ClientCache} uses {@link SocketFactory} as the cache key.
+ * The default {@link StandardSocketFactory#hashCode()} hashes to the class name, which
+ * means that all clients with the same socket factory share the same {@link org.apache.hadoop.ipc.Client}.
+ * However, sometimes we want to have separate client with different timeout. In that case,
+ * we need to have a separate {@link SocketFactory}.
+ * @param protocol SCM RPC protocol
+ * @param conf Configuration
+ * @param userGroupInformation custom ugi for this client. If not specified the implementation defaults to
+ * {@link UserGroupInformation#getCurrentUser()}
+ * @param socketFactory custom socket factory for the client. If not specified, the implement defaults to
+ * {@link NetUtils#getDefaultSocketFactory(Configuration)}
*/
public SCMFailoverProxyProviderBase(Class Hadoop's {@link ClientCache} uses a {@link SocketFactory} as the cache key
+ * and deduplicates IPC clients when socket factories compare equal. Wrapping a
+ * factory in this class provides a distinct cache key so a client can use
+ * configuration that must not be shared with other Hadoop IPC clients.
+ *
+ * Unlike {@link StandardSocketFactory}, this class intentionally does not
+ * implement {@link Object#equals(Object)} or {@link Object#hashCode()}.
+ * Retaining {@link Object}'s identity-based implementations ensures that each
+ * instance remains a distinct Hadoop IPC client cache key.
+ */
+final class IdentitySocketFactory extends SocketFactory {
+ private final SocketFactory delegate;
+
+ IdentitySocketFactory(SocketFactory delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public Socket createSocket() throws IOException {
+ return delegate.createSocket();
+ }
+
+ @Override
+ public Socket createSocket(String host, int port) throws IOException {
+ return delegate.createSocket(host, port);
+ }
+
+ @Override
+ public Socket createSocket(String host, int port, InetAddress localHost,
+ int localPort) throws IOException {
+ return delegate.createSocket(host, port, localHost, localPort);
+ }
+
+ @Override
+ public Socket createSocket(InetAddress host, int port) throws IOException {
+ return delegate.createSocket(host, port);
+ }
+
+ @Override
+ public Socket createSocket(InetAddress address, int port,
+ InetAddress localAddress, int localPort) throws IOException {
+ return delegate.createSocket(address, port, localAddress, localPort);
+ }
+}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 5852e06ea754..ab2262598f2f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -55,6 +55,34 @@ public final class OMConfigKeys {
public static final int OZONE_OM_DB_MAX_OPEN_FILES_DEFAULT
= -1;
+ public static final String OZONE_OM_SCM_LOCATION_CLIENT_RPC_TIMEOUT =
+ "ozone.om.scmclient.location.rpc.timeout";
+ public static final String OZONE_OM_SCM_LOCATION_CLIENT_RPC_TIMEOUT_DEFAULT =
+ "30s";
+ public static final String OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT =
+ "ozone.om.scmclient.location.ipc.connect.timeout";
+ public static final String
+ OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_DEFAULT =
+ "5s";
+ public static final String
+ OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_RETRIES =
+ "ozone.om.scmclient.location.ipc.connect.max.retries.on.timeouts";
+ public static final String
+ OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_RETRIES_DEFAULT = "0";
+ public static final String OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY =
+ "ozone.om.scmclient.location.failover.max.retry";
+ public static final String
+ OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY_DEFAULT = "3";
+ public static final String OZONE_OM_SCM_LOCATION_CLIENT_MAX_RETRY_TIMEOUT =
+ "ozone.om.scmclient.location.max.retry.timeout";
+ public static final String
+ OZONE_OM_SCM_LOCATION_CLIENT_MAX_RETRY_TIMEOUT_DEFAULT = "6s";
+ public static final String
+ OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_RETRY_INTERVAL =
+ "ozone.om.scmclient.location.failover.retry.interval";
+ public static final String
+ OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_RETRY_INTERVAL_DEFAULT = "2s";
+
public static final String OZONE_OM_INTERNAL_SERVICE_ID =
"ozone.om.internal.service.id";
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMScmLocationClientConfig.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMScmLocationClientConfig.java
new file mode 100644
index 000000000000..4fff70789ec2
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMScmLocationClientConfig.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_RETRY_INTERVAL;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_RETRY_INTERVAL_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_RETRIES;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_RETRIES_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_MAX_RETRY_TIMEOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_MAX_RETRY_TIMEOUT_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_RPC_TIMEOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_RPC_TIMEOUT_DEFAULT;
+
+import java.util.concurrent.TimeUnit;
+import javax.net.SocketFactory;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+
+/**
+ * Builds the configuration used by OM's SCM location clients.
+ *
+ * The default effective retry limit is the largest of the configured retry
+ * count, retry timeout divided by retry interval, and configured SCM node
+ * count:
+ * Hadoop tracks retries and failovers separately. A sequence of {@code R}
+ * retry-without-failover responses followed by {@code R} failover failures
+ * can therefore make {@code 2R + 1} attempts. Its configured timeout
+ * envelopes are:
+ *
+ * R = max(3, 6s / 2s, SCM node count)
+ *
+ * For failures that always trigger failover, this allows {@code R + 1}
+ * attempts. The configured timeout envelopes are:
+ *
+ * established connection: (R + 1) * 30s + R * 2s
+ * including TCP connect: (R + 1) * (5s + 30s) + R * 2s
+ *
+ *
+ *
+ * established connection: (2R + 1) * 30s + 2R * 2s
+ * including TCP connect: (2R + 1) * (5s + 30s) + 2R * 2s
+ *
+ * Actual calls can complete sooner. These calculations are not caller-side
+ * deadlines and exclude time outside the configured connect and RPC waits.
+ */
+public final class OMScmLocationClientConfig {
+ private OMScmLocationClientConfig() {
+ }
+
+ public static OzoneConfiguration createScmClientConfiguration(
+ OzoneConfiguration configuration) {
+ OzoneConfiguration scmClientConfiguration =
+ new OzoneConfiguration(configuration);
+ copy(configuration, scmClientConfiguration,
+ OZONE_OM_SCM_LOCATION_CLIENT_RPC_TIMEOUT,
+ OZONE_OM_SCM_LOCATION_CLIENT_RPC_TIMEOUT_DEFAULT,
+ OzoneConfigKeys.HDDS_SCM_CLIENT_RPC_TIME_OUT);
+ copyTimeDurationInMillis(configuration, scmClientConfiguration,
+ OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT,
+ OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_DEFAULT,
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY);
+ copy(configuration, scmClientConfiguration,
+ OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_RETRIES,
+ OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_RETRIES_DEFAULT,
+ CommonConfigurationKeysPublic
+ .IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY);
+ copy(configuration, scmClientConfiguration,
+ OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY,
+ OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY_DEFAULT,
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY);
+ String scmServiceId = HddsUtils.getScmServiceId(configuration);
+ int scmNodeCount = scmServiceId == null ? 1 :
+ HddsUtils.getSCMNodeIds(configuration, scmServiceId).size();
+ int retryCount = scmClientConfiguration.getInt(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY, 0);
+ scmClientConfiguration.setInt(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY,
+ Math.max(retryCount, scmNodeCount));
+ copy(configuration, scmClientConfiguration,
+ OZONE_OM_SCM_LOCATION_CLIENT_MAX_RETRY_TIMEOUT,
+ OZONE_OM_SCM_LOCATION_CLIENT_MAX_RETRY_TIMEOUT_DEFAULT,
+ OzoneConfigKeys.HDDS_SCM_CLIENT_MAX_RETRY_TIMEOUT);
+ copy(configuration, scmClientConfiguration,
+ OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_RETRY_INTERVAL,
+ OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_RETRY_INTERVAL_DEFAULT,
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_RETRY_INTERVAL);
+ return scmClientConfiguration;
+ }
+
+ /**
+ * Creates a socket factory with a separate Hadoop IPC client cache entry so
+ * the SCM location connection timeout is not shared with other clients.
+ */
+ static SocketFactory createSocketFactory(OzoneConfiguration configuration) {
+ return new IdentitySocketFactory(
+ NetUtils.getDefaultSocketFactory(configuration));
+ }
+
+ private static void copy(OzoneConfiguration source,
+ OzoneConfiguration target, String sourceKey, String defaultValue,
+ String targetKey) {
+ target.set(targetKey, source.get(sourceKey, defaultValue));
+ }
+
+ private static void copyTimeDurationInMillis(OzoneConfiguration source,
+ OzoneConfiguration target, String sourceKey, String defaultValue,
+ String targetKey) {
+ long value = source.getTimeDuration(sourceKey, defaultValue,
+ TimeUnit.MILLISECONDS);
+ target.setInt(targetKey, Math.toIntExact(value));
+ }
+}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/TestIdentitySocketFactory.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/TestIdentitySocketFactory.java
new file mode 100644
index 000000000000..e0139d45d8a9
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/TestIdentitySocketFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+
+import javax.net.SocketFactory;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.io.ObjectWritable;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ipc.ClientCache;
+import org.apache.hadoop.net.NetUtils;
+import org.junit.jupiter.api.Test;
+
+/** Tests {@link IdentitySocketFactory}. */
+class TestIdentitySocketFactory {
+
+ @Test
+ void usesIdentityForHadoopClientCacheKeys() {
+ OzoneConfiguration configuration = new OzoneConfiguration();
+ SocketFactory delegate = NetUtils.getDefaultSocketFactory(configuration);
+ SocketFactory firstFactory = new IdentitySocketFactory(delegate);
+ SocketFactory secondFactory = new IdentitySocketFactory(delegate);
+ ClientCache clientCache = new ClientCache();
+ Client firstClient = clientCache.getClient(configuration, firstFactory,
+ ObjectWritable.class);
+ Client secondClient = clientCache.getClient(configuration, secondFactory,
+ ObjectWritable.class);
+
+ try {
+ assertNotSame(firstClient, secondClient);
+ } finally {
+ clientCache.stopClient(firstClient);
+ clientCache.stopClient(secondClient);
+ }
+ }
+}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/TestOMScmLocationClientConfig.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/TestOMScmLocationClientConfig.java
new file mode 100644
index 000000000000..d41b586c33b2
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/TestOMScmLocationClientConfig.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NODES_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_RETRY_INTERVAL;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_RETRIES;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_MAX_RETRY_TIMEOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SCM_LOCATION_CLIENT_RPC_TIMEOUT;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+
+import java.util.concurrent.TimeUnit;
+import javax.net.SocketFactory;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.io.ObjectWritable;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ipc.ClientCache;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.junit.jupiter.api.Test;
+
+/** Tests OM-specific SCM client configuration. */
+class TestOMScmLocationClientConfig {
+ @Test
+ void overridesScmClientAndIpcTimeoutsWithoutMutatingSourceConfiguration() {
+ OzoneConfiguration configuration = new OzoneConfiguration();
+ configuration.setTimeDuration(OzoneConfigKeys.HDDS_SCM_CLIENT_RPC_TIME_OUT,
+ 15, TimeUnit.MINUTES);
+ configuration.setTimeDuration(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,
+ 20, TimeUnit.SECONDS);
+ configuration.setInt(
+ CommonConfigurationKeysPublic
+ .IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
+ 45);
+ configuration.setInt(OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY,
+ 15);
+ configuration.setTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_MAX_RETRY_TIMEOUT,
+ 10, TimeUnit.MINUTES);
+ configuration.setTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_RETRY_INTERVAL,
+ 9, TimeUnit.SECONDS);
+ configuration.setTimeDuration(OZONE_OM_SCM_LOCATION_CLIENT_RPC_TIMEOUT,
+ 60, TimeUnit.SECONDS);
+ configuration.setTimeDuration(
+ OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT,
+ 7, TimeUnit.SECONDS);
+ configuration.setInt(
+ OZONE_OM_SCM_LOCATION_CLIENT_IPC_CONNECT_TIMEOUT_RETRIES, 1);
+ configuration.setInt(OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY, 4);
+ configuration.setTimeDuration(
+ OZONE_OM_SCM_LOCATION_CLIENT_MAX_RETRY_TIMEOUT,
+ 8, TimeUnit.SECONDS);
+ configuration.setTimeDuration(
+ OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_RETRY_INTERVAL,
+ 1, TimeUnit.SECONDS);
+
+ OzoneConfiguration scmClientConfiguration =
+ OMScmLocationClientConfig.createScmClientConfiguration(configuration);
+
+ assertEquals(60, scmClientConfiguration.getTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_RPC_TIME_OUT, 0, TimeUnit.SECONDS));
+ assertEquals(7_000, scmClientConfiguration.getInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY, 0));
+ assertEquals(1, scmClientConfiguration.getInt(
+ CommonConfigurationKeysPublic
+ .IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
+ 0));
+ assertEquals(4, scmClientConfiguration.getInt(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY, 0));
+ assertEquals(8, scmClientConfiguration.getTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_MAX_RETRY_TIMEOUT,
+ 0, TimeUnit.SECONDS));
+ assertEquals(1, scmClientConfiguration.getTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_RETRY_INTERVAL,
+ 0, TimeUnit.SECONDS));
+ assertEquals(15, configuration.getTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_RPC_TIME_OUT, 0, TimeUnit.MINUTES));
+ assertEquals(20, configuration.getTimeDuration(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,
+ 0, TimeUnit.SECONDS));
+ assertEquals(45, configuration.getInt(
+ CommonConfigurationKeysPublic
+ .IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
+ 0));
+ assertEquals(15, configuration.getInt(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY, 0));
+ assertEquals(10, configuration.getTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_MAX_RETRY_TIMEOUT,
+ 0, TimeUnit.MINUTES));
+ assertEquals(9, configuration.getTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_RETRY_INTERVAL,
+ 0, TimeUnit.SECONDS));
+ }
+
+ @Test
+ void appliesOmDefaultsWhenOverridesAreNotSet() {
+ OzoneConfiguration configuration = new OzoneConfiguration();
+ configuration.setTimeDuration(OzoneConfigKeys.HDDS_SCM_CLIENT_RPC_TIME_OUT,
+ 2, TimeUnit.MINUTES);
+ configuration.setTimeDuration(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,
+ 10, TimeUnit.SECONDS);
+ configuration.setInt(
+ CommonConfigurationKeysPublic
+ .IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
+ 3);
+ configuration.setInt(OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY,
+ 15);
+ configuration.setTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_MAX_RETRY_TIMEOUT,
+ 10, TimeUnit.MINUTES);
+ configuration.setTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_RETRY_INTERVAL,
+ 9, TimeUnit.SECONDS);
+
+ OzoneConfiguration scmClientConfiguration =
+ OMScmLocationClientConfig.createScmClientConfiguration(configuration);
+
+ assertEquals(30, scmClientConfiguration.getTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_RPC_TIME_OUT, 0, TimeUnit.SECONDS));
+ assertEquals(5_000, scmClientConfiguration.getInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY, 0));
+ assertEquals(0, scmClientConfiguration.getInt(
+ CommonConfigurationKeysPublic
+ .IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
+ 0));
+ assertEquals(3, scmClientConfiguration.getInt(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY, 0));
+ assertEquals(6, scmClientConfiguration.getTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_MAX_RETRY_TIMEOUT,
+ 0, TimeUnit.SECONDS));
+ assertEquals(2, scmClientConfiguration.getTimeDuration(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_RETRY_INTERVAL,
+ 0, TimeUnit.SECONDS));
+ }
+
+ @Test
+ void boundsRetryCountByConfiguredScmNodes() {
+ OzoneConfiguration configuration = new OzoneConfiguration();
+ String scmServiceId = "scmservice";
+ configuration.set(OZONE_SCM_SERVICE_IDS_KEY, scmServiceId);
+ configuration.set(OZONE_SCM_NODES_KEY + "." + scmServiceId,
+ "scm1,scm2,scm3,scm4,scm5,scm6");
+ configuration.setInt(OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY, 3);
+
+ OzoneConfiguration scmClientConfiguration =
+ OMScmLocationClientConfig.createScmClientConfiguration(configuration);
+
+ assertEquals(6, scmClientConfiguration.getInt(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY, 0));
+ assertEquals(3, configuration.getInt(
+ OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY, 0));
+
+ configuration.setInt(OZONE_OM_SCM_LOCATION_CLIENT_FAILOVER_MAX_RETRY, 8);
+ scmClientConfiguration =
+ OMScmLocationClientConfig.createScmClientConfiguration(configuration);
+ assertEquals(8, scmClientConfiguration.getInt(
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY, 0));
+ }
+
+ @Test
+ void usesAnIpcClientSeparateFromTheDefaultSocketFactory() {
+ OzoneConfiguration configuration = new OzoneConfiguration();
+ SocketFactory defaultSocketFactory =
+ NetUtils.getDefaultSocketFactory(configuration);
+ SocketFactory scmLocationSocketFactory =
+ OMScmLocationClientConfig.createSocketFactory(configuration);
+ ClientCache clientCache = new ClientCache();
+ Client defaultClient = clientCache.getClient(configuration,
+ defaultSocketFactory, ObjectWritable.class);
+ Client criticalScmClient = clientCache.getClient(configuration,
+ scmLocationSocketFactory, ObjectWritable.class);
+
+ try {
+ assertNotSame(defaultClient, criticalScmClient);
+ } finally {
+ clientCache.stopClient(defaultClient);
+ clientCache.stopClient(criticalScmClient);
+ }
+ }
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 418c62a1d12e..79965a993960 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -187,8 +187,8 @@ private void addPropertiesNotInXml() {
DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
OzoneConfigKeys.HDDS_SCM_CLIENT_RPC_TIME_OUT,
OzoneConfigKeys.HDDS_SCM_CLIENT_MAX_RETRY_TIMEOUT,
- OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY,
+ OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_RETRY_INTERVAL
));
}
}
-
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index d74614579e24..dd12a45056bd 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -280,7 +280,7 @@ public void start(OzoneConfiguration configuration) {
keyDeletingServiceCorePoolSize = 1;
}
keyDeletingService = new KeyDeletingService(ozoneManager,
- scmClient.getBlockClient(), blockDeleteInterval,
+ scmClient.getBlockClientForKeyDeletion(), blockDeleteInterval,
serviceTimeout, configuration, keyDeletingServiceCorePoolSize, isSnapshotDeepCleaningEnabled);
keyDeletingService.start();
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 1960e1fc3bdc..1a68c7396d30 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -165,6 +165,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.management.ObjectName;
+import javax.net.SocketFactory;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.Configuration;
@@ -660,12 +661,22 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
// Honor property 'hadoop.security.token.service.use_ip'
omRpcAddressTxt = new Text(SecurityUtil.buildTokenService(omNodeRpcAddr));
- final StorageContainerLocationProtocol scmContainerClient = getScmContainerClient(configuration);
+ OzoneConfiguration scmLocationClientConfiguration =
+ OMScmLocationClientConfig.createScmClientConfiguration(configuration);
+ SocketFactory scmLocationSocketFactory =
+ OMScmLocationClientConfig.createSocketFactory(configuration);
+ StorageContainerLocationProtocol scmContainerClient =
+ getScmContainerClient(scmLocationClientConfiguration,
+ scmLocationSocketFactory);
// verifies that the SCM info in the OM Version file is correct.
- final ScmBlockLocationProtocol scmBlockClient = getScmBlockClient(configuration);
+ ScmBlockLocationProtocol scmBlockClient =
+ getScmBlockClient(scmLocationClientConfiguration,
+ scmLocationSocketFactory);
+ ScmBlockLocationProtocol keyDeletionScmBlockClient =
+ getScmBlockClient(configuration, null);
scmTopologyClient = new ScmTopologyClient(scmBlockClient);
this.scmClient = new ScmClient(scmBlockClient, scmContainerClient,
- configuration);
+ configuration, keyDeletionScmBlockClient);
this.ozoneLockProvider = new OzoneLockProvider(getKeyPathLockEnabled(),
getEnableFileSystemPaths());
@@ -1512,13 +1523,13 @@ private static void loginOMUser(OzoneConfiguration conf)
}
/**
- * Create a scm block client, used by putKey() and getKey().
+ * Creates a scm block client.
*
* @return {@link ScmBlockLocationProtocol}
*/
private static ScmBlockLocationProtocol getScmBlockClient(
- OzoneConfiguration conf) {
- return HAUtils.getScmBlockClient(conf);
+ OzoneConfiguration conf, SocketFactory socketFactory) {
+ return HAUtils.getScmBlockClient(conf, socketFactory);
}
/**
@@ -1527,8 +1538,8 @@ private static ScmBlockLocationProtocol getScmBlockClient(
* @return {@link StorageContainerLocationProtocol}
*/
private static StorageContainerLocationProtocol getScmContainerClient(
- OzoneConfiguration conf) {
- return HAUtils.getScmContainerClient(conf);
+ OzoneConfiguration conf, SocketFactory socketFactory) {
+ return HAUtils.getScmContainerClient(conf, null, socketFactory);
}
/**
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java
index a61595faf069..5b86ad965458 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java
@@ -54,6 +54,7 @@
public class ScmClient {
private final ScmBlockLocationProtocol blockClient;
+ private final ScmBlockLocationProtocol blockClientForKeyDeletion;
private final StorageContainerLocationProtocol containerClient;
private final LoadingCache