fix(management-network): support target-aware endpoints [ZSTAC-86567]

conf/salt/master

@@ -12,11 +12,11 @@
 #
 
 # The address of the interface to bind to
-#interface: 0.0.0.0
+interface: {masterInterface}
 
 # Whether the master should listen for IPv6 connections. If this is set to True,
 # the interface option must be adjusted too (for example: "interface: '::'")
-#ipv6: False
+ipv6: {ipv6Enabled}
 
 # The tcp port used by the publisher
 #publish_port: 4505

conf/salt/minion

@@ -12,7 +12,7 @@
 master: {managementNodeIp}
 
 # Set whether the minion should connect to the master via IPv6
-#ipv6: False
+ipv6: {ipv6Enabled}
 
 # Set the number of seconds to wait before attempting to resolve
 # the master hostname if name resolution fails. Defaults to 30 seconds.

core/src/main/java/org/zstack/core/ManagementServerIpSelection.java

@@ -0,0 +1,78 @@
+package org.zstack.core;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class ManagementServerIpSelection {
+    public enum FailureReason {
+        NO_SAME_FAMILY_MGT,
+        ROUTE_LOOKUP_FAILED,
+        ROUTE_SOURCE_NOT_MANAGEMENT
+    }
+
+    private final String selectedIp;
+    private final String remoteIp;
+    private final int remoteFamily;
+    private final List<String> managementServerIps;
+    private final String routeSourceIp;
+    private final String routeError;
+    private final FailureReason failureReason;
+
+    private ManagementServerIpSelection(String selectedIp, String remoteIp, int remoteFamily,
+                                        List<String> managementServerIps, String routeSourceIp,
+                                        String routeError, FailureReason failureReason) {
+        this.selectedIp = selectedIp;
+        this.remoteIp = remoteIp;
+        this.remoteFamily = remoteFamily;
+        this.managementServerIps = Collections.unmodifiableList(new ArrayList<>(managementServerIps));
+        this.routeSourceIp = routeSourceIp;
+        this.routeError = routeError;
+        this.failureReason = failureReason;
+    }
+
+    public static ManagementServerIpSelection success(String selectedIp, String remoteIp, int remoteFamily,
+                                                       List<String> managementServerIps, String routeSourceIp) {
+        return new ManagementServerIpSelection(selectedIp, remoteIp, remoteFamily,
+                managementServerIps, routeSourceIp, null, null);
+    }
+
+    public static ManagementServerIpSelection failure(String remoteIp, int remoteFamily,
+                                                       List<String> managementServerIps, String routeSourceIp,
+                                                       String routeError, FailureReason failureReason) {
+        return new ManagementServerIpSelection(null, remoteIp, remoteFamily,
+                managementServerIps, routeSourceIp, routeError, failureReason);
+    }
+
+    public boolean isSuccess() {
+        return failureReason == null;
+    }
+
+    public String getSelectedIp() {
+        return selectedIp;
+    }
+
+    public String getRemoteIp() {
+        return remoteIp;
+    }
+
+    public int getRemoteFamily() {
+        return remoteFamily;
+    }
+
+    public List<String> getManagementServerIps() {
+        return managementServerIps;
+    }
+
+    public String getRouteSourceIp() {
+        return routeSourceIp;
+    }
+
+    public String getRouteError() {
+        return routeError;
+    }
+
+    public FailureReason getFailureReason() {
+        return failureReason;
+    }
+}

core/src/main/java/org/zstack/core/Platform.java

@@ -75,6 +75,9 @@
 import static org.zstack.utils.CollectionDSL.e;
 import static org.zstack.utils.CollectionDSL.map;
 import static org.zstack.utils.StringDSL.ln;
+import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000;
+import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001;
+import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002;
 
 public class Platform {
     private static final CLogger logger = CLoggerImpl.getLogger(Platform.class);
@@ -1391,6 +1394,125 @@ public static String getManagementServerIpForRemote(String remoteIp) {
         return selectManagementServerIpForRemote(remoteIp, null);
     }
 
+    public static ManagementServerIpSelection resolveManagementServerIpForRemoteStrict(String remoteIp) {
+        String normalizedRemoteIp = normalizeManagementIp(remoteIp);
+        int remoteFamily = getIpFamily(normalizedRemoteIp);
+        List<String> managementServerIps = getManagementServerIps().stream()
+                .map(Platform::normalizeManagementIp)
+                .distinct()
+                .collect(Collectors.toList());
+        if (remoteFamily == 0) {
+            return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily,
+                    managementServerIps, null, "remote target is not an IP address",
+                    ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED);
+        }
+        boolean hasSameFamily = managementServerIps.stream()
+                .anyMatch(ip -> getIpFamily(ip) == remoteFamily);
+        if (!hasSameFamily) {
+            return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily,
+                    managementServerIps, null, null,
+                    ManagementServerIpSelection.FailureReason.NO_SAME_FAMILY_MGT);
+        }
+
+        String family = remoteFamily == IPv6Constants.IPv6 ? "-6" : "-4";
+        Linux.ShellResult ret = Linux.shell(String.format("/sbin/ip %s route get %s", family, normalizedRemoteIp));
+        if (ret.getExitCode() != 0) {
+            String routeError = String.format("exitCode=%s, stdout=%s, stderr=%s",
+                    ret.getExitCode(), ret.getStdout(), ret.getStderr());
+            return selectManagementServerIpForRemoteStrict(normalizedRemoteIp, null, routeError);
+        }
+
+        String routeSourceIp = parseRouteSourceIp(normalizedRemoteIp, ret.getStdout());
+        if (routeSourceIp == null) {
+            return selectManagementServerIpForRemoteStrict(normalizedRemoteIp, null,
+                    String.format("route output has no same-family source: %s", ret.getStdout()));
+        }
+        return selectManagementServerIpForRemoteStrict(normalizedRemoteIp, routeSourceIp, null);
+    }
+
+    public static ManagementServerIpSelection selectManagementServerIpForRemoteStrict(
+            String remoteIp, String routeSourceIp, String routeError) {
+        String normalizedRemoteIp = normalizeManagementIp(remoteIp);
+        String normalizedRouteSourceIp = normalizeManagementIp(routeSourceIp);
+        int remoteFamily = getIpFamily(normalizedRemoteIp);
+        List<String> managementServerIps = getManagementServerIps().stream()
+                .map(Platform::normalizeManagementIp)
+                .distinct()
+                .collect(Collectors.toList());
+        List<String> sameFamilyIps = managementServerIps.stream()
+                .filter(ip -> getIpFamily(ip) == remoteFamily)
+                .collect(Collectors.toList());
+
+        if (remoteFamily == 0) {
+            return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily,
+                    managementServerIps, normalizedRouteSourceIp, "remote target is not an IP address",
+                    ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED);
+        }
+        if (sameFamilyIps.isEmpty()) {
+            return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily,
+                    managementServerIps, normalizedRouteSourceIp, routeError,
+                    ManagementServerIpSelection.FailureReason.NO_SAME_FAMILY_MGT);
+        }
+        if (StringUtils.isBlank(normalizedRouteSourceIp)) {
+            String details = StringUtils.isBlank(routeError) ? "route source is unavailable" : routeError;
+            return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily,
+                    managementServerIps, null, details,
+                    ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED);
+        }
+        if (getIpFamily(normalizedRouteSourceIp) != remoteFamily ||
+                !sameFamilyIps.contains(normalizedRouteSourceIp)) {
+            return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily,
+                    managementServerIps, normalizedRouteSourceIp, routeError,
+                    ManagementServerIpSelection.FailureReason.ROUTE_SOURCE_NOT_MANAGEMENT);
+        }
+        return ManagementServerIpSelection.success(normalizedRouteSourceIp, normalizedRemoteIp,
+                remoteFamily, managementServerIps, normalizedRouteSourceIp);
+    }
+
+    public static ErrorCode managementServerIpSelectionError(ManagementServerIpSelection selection) {
+        if (selection == null || selection.isSuccess()) {
+            throw new IllegalArgumentException("selection must contain a failure");
+        }
+        String context = String.format("remote[%s], family[IPv%s], managementIps%s, routeSource[%s], routeError[%s]",
+                selection.getRemoteIp(), selection.getRemoteFamily(), selection.getManagementServerIps(),
+                selection.getRouteSourceIp(), selection.getRouteError());
+        switch (selection.getFailureReason()) {
+            case NO_SAME_FAMILY_MGT:
+                return operr(ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000,
+                        "cannot find a same-family management server IP for %s", context);
+            case ROUTE_LOOKUP_FAILED:
+                return operr(ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001,
+                        "failed to resolve the management server route source for %s", context);
+            case ROUTE_SOURCE_NOT_MANAGEMENT:
+                return operr(ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002,
+                        "route source is not a configured management server IP for %s", context);
+            default:
+                throw new IllegalArgumentException("selection is successful or has no failure reason");
+        }
+    }
+
+    private static int getIpFamily(String ip) {
+        if (IPv6NetworkUtils.isIpv6Address(ip)) {
+            return IPv6Constants.IPv6;
+        }
+        if (NetworkUtils.isIpv4Address(ip)) {
+            return IPv6Constants.IPv4;
+        }
+        return 0;
+    }
+
+    private static String parseRouteSourceIp(String remoteIp, String output) {
+        String[] tokens = StringUtils.defaultString(output).trim().split("\\s+");
+        int remoteFamily = getIpFamily(remoteIp);
+        for (int i = 0; i < tokens.length - 1; i++) {
+            if ("src".equals(tokens[i])) {
+                String sourceIp = normalizeManagementIp(tokens[i + 1]);
+                return getIpFamily(sourceIp) == remoteFamily ? sourceIp : null;
+            }
+        }
+        return null;
+    }
+
     public static String selectManagementServerIpForRemote(String remoteIp, String routeSourceIp) {
         if (StringUtils.isBlank(remoteIp)) {
             return getManagementServerIp();

core/src/main/java/org/zstack/core/agent/AgentManagerImpl.java

@@ -3,6 +3,8 @@
 import org.apache.commons.io.FileUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.zstack.core.CoreGlobalProperty;
+import org.zstack.core.ManagementServerIpSelection;
+import org.zstack.core.Platform;
 import org.zstack.core.ansible.AnsibleNeedRun;
 import org.zstack.core.ansible.AnsibleRunner;
 import org.zstack.core.ansible.SshFolderMd5Checker;
@@ -52,6 +54,11 @@ public static String buildAgentUrl(String ip, int port, String path) {
         return String.format(HTTP_URL_FORMAT, IPv6NetworkUtils.formatHostForUrl(ip), port, path);
     }
 
+    public static String buildCommandUrl(RESTFacade restf, ManagementServerIpSelection selection) {
+        return selection == null ? restf.getSendCommandUrl() :
+                restf.buildSendCommandUrlForManagementHost(selection.getSelectedIp());
+    }
+
     @Autowired
     private CloudBus bus;
     @Autowired
@@ -114,7 +121,8 @@ public String getName() {
         });
     }
 
-    private void connect(final DeployAgentMsg msg, final Completion completion) {
+    private void connect(final DeployAgentMsg msg, final ManagementServerIpSelection managementServerIpSelection,
+                         final Completion completion) {
         FlowChain chain = FlowChainBuilder.newShareFlowChain();
         chain.setName(String.format("continue-connect-agent-server-%s:%s", msg.getIp(), msg.getAgentPort()));
         chain.then(new ShareFlow() {
@@ -149,7 +157,8 @@ public void fail(ErrorCode errorCode) {
                     @Override
                     public void run(FlowTrigger trigger, Map data) {
                         Map<String, Object> config = new HashMap<String, Object>();
-                        config.put(AgentConstant.CONFIG_COMMAND_URL, restf.getSendCommandUrl());
+                        config.put(AgentConstant.CONFIG_COMMAND_URL,
+                                buildCommandUrl(restf, managementServerIpSelection));
                         if (msg.getConfig() != null) {
                             config.putAll(msg.getConfig());
                         }
@@ -193,6 +202,19 @@ private void deployAgent(final DeployAgentMsg msg, final NoErrorCompletion noErr
             return;
         }
 
+        final ManagementServerIpSelection managementServerIpSelection;
+        if (NetworkUtils.isIpAddress(msg.getIp())) {
+            managementServerIpSelection = Platform.resolveManagementServerIpForRemoteStrict(msg.getIp());
+            if (!managementServerIpSelection.isSuccess()) {
+                reply.setError(Platform.managementServerIpSelectionError(managementServerIpSelection));
+                bus.reply(msg, reply);
+                noErrorCompletion.done();
+                return;
+            }
+        } else {
+            managementServerIpSelection = null;
+        }
+
         try {
             String agentYamlPath = PathUtil.join(AgentConstant.ANSIBLE_MODULE_PATH, "server", AgentConstant.ANSIBLE_PLAYBOOK_NAME);
             File agentYaml = PathUtil.findFileOnClassPath(agentYamlPath, true);
@@ -254,6 +276,7 @@ public boolean isRunNeed() {
                 runner.setSshPort(msg.getSshPort());
             }
             runner.setTargetIp(msg.getIp());
+            runner.setManagementServerIpSelection(managementServerIpSelection);
             runner.setPlayBookPath(tmpAgentYaml.getAbsolutePath());
             runner.run(new ReturnValueCompletion<Boolean>(msg, noErrorCompletion) {
                 @Override
@@ -271,7 +294,7 @@ public void run() {
                         }
                     });
 
-                    connect(msg, new Completion(msg, noErrorCompletion) {
+                    connect(msg, managementServerIpSelection, new Completion(msg, noErrorCompletion) {
                         @Override
                         public void success() {
                             bus.reply(msg, reply);

core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java

@@ -4,6 +4,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Configurable;
 import org.zstack.core.CoreGlobalProperty;
+import org.zstack.core.Platform;
+import org.zstack.core.ManagementServerIpSelection;
 import org.zstack.core.cloudbus.CloudBus;
 import org.zstack.core.cloudbus.CloudBusCallBack;
 import org.zstack.header.core.Completion;
@@ -84,6 +86,7 @@ public class AnsibleRunner {
     private boolean forceRun;
 
     private AnsibleBasicArguments deployArguments;
+    private ManagementServerIpSelection managementServerIpSelection;
 
     public String getAnsibleExecutable() {
         return ansibleExecutable;
@@ -197,6 +200,10 @@ public void setDeployArguments(AnsibleBasicArguments deployArguments) {
         this.deployArguments = deployArguments;
     }
 
+    public void setManagementServerIpSelection(ManagementServerIpSelection selection) {
+        this.managementServerIpSelection = selection;
+    }
+
     public Map<String, Object> getArguments() {
         return arguments;
     }
@@ -382,6 +389,24 @@ private void cleanup() {
 
     public void run(ReturnValueCompletion<Boolean> completion) {
         try {
+            String deploymentHost;
+            if (NetworkUtils.isIpAddress(targetIp)) {
+                ManagementServerIpSelection selection = managementServerIpSelection == null ?
+                        Platform.resolveManagementServerIpForRemoteStrict(targetIp) : managementServerIpSelection;
+                if (!selection.isSuccess()) {
+                    completion.fail(Platform.managementServerIpSelectionError(selection));
+                    return;
+                }
+                deploymentHost = selection.getSelectedIp();
+            } else {
+                deploymentHost = restf.getHostName();
+            }
+            for (AnsibleChecker checker : checkers) {
+                if (checker instanceof ManagementServerIpAwareChecker) {
+                    ((ManagementServerIpAwareChecker) checker).setManagementServerIp(deploymentHost);
+                }
+            }
+
             if (!forceRun && !isNeedRun()) {
                 completion.success(false);
                 return;
@@ -393,9 +418,9 @@ public void run(ReturnValueCompletion<Boolean> completion) {
                 deployArguments = new AnsibleBasicArguments();
             }
 
-            deployArguments.setPipUrl(buildPipUrl(restf.getHostName(), port));
-            deployArguments.setTrustedHost(restf.getHostName());
-            deployArguments.setYumServer(IPv6NetworkUtils.formatHostPort(restf.getHostName(), port));
+            deployArguments.setPipUrl(buildPipUrl(deploymentHost, port));
+            deployArguments.setTrustedHost(deploymentHost);
+            deployArguments.setYumServer(IPv6NetworkUtils.formatHostPort(deploymentHost, port));
             deployArguments.setRemoteUser(username);
             if (password != null && !password.isEmpty()) {
                 deployArguments.setRemotePass(password);