From ad7511b9959f3867c604b7ff70ceb5aa1e328b00 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Sat, 11 Jul 2026 17:18:16 +0900 Subject: [PATCH] [management-network]: select target-aware MN endpoints Select the exact configured management IP chosen by the kernel route for each literal agent target. Return structured errors for missing family, route lookup failure, and unconfigured route source. Keep hostname compatibility and avoid host flapping on transient ping route lookup failures. Resolves: ZSTAC-86567 Change-Id: I9d3f0ea84d6a4eeb99c5a055c4fbd9003eefbe18 --- conf/salt/master | 4 +- conf/salt/minion | 2 +- .../core/ManagementServerIpSelection.java | 78 +++ .../main/java/org/zstack/core/Platform.java | 122 +++++ .../zstack/core/agent/AgentManagerImpl.java | 29 +- .../zstack/core/ansible/AnsibleRunner.java | 31 +- .../core/ansible/CallBackNetworkChecker.java | 7 +- .../ManagementServerIpAwareChecker.java | 5 + .../core/ansible/SshYumRepoChecker.java | 82 +++- .../org/zstack/core/rest/RESTFacadeImpl.java | 31 +- .../org/zstack/core/salt/SaltFacadeImpl.java | 24 +- .../zstack/core/salt/SaltSetupMinionJob.java | 39 +- .../org/zstack/header/rest/RESTFacade.java | 4 + .../src/main/java/org/zstack/kvm/KVMHost.java | 154 +++++- .../agent/AgentManagerCommandUrlCase.groovy | 30 ++ .../core/ansible/SshYumRepoCheckerCase.groovy | 49 ++ .../core/ManagementNetworkIpv6Case.groovy | 92 +++- ...sibleRunnerTargetAwareArgumentsCase.groovy | 256 ++++++++++ .../core/salt/SaltDualStackConfigCase.groovy | 112 +++++ .../kvm/host/DualStackAddKvmHostCase.groovy | 463 ++++++++++++++++++ .../MigrateVmFromDestitonHostCase.groovy | 56 ++- .../java/org/zstack/utils/IptablesUtils.java | 12 +- .../CloudOperationsErrorCode.java | 7 + 23 files changed, 1630 insertions(+), 59 deletions(-) create mode 100644 core/src/main/java/org/zstack/core/ManagementServerIpSelection.java create mode 100644 core/src/main/java/org/zstack/core/ansible/ManagementServerIpAwareChecker.java create mode 100644 test/src/test/groovy/org/zstack/core/agent/AgentManagerCommandUrlCase.groovy create mode 100644 test/src/test/groovy/org/zstack/core/ansible/SshYumRepoCheckerCase.groovy create mode 100644 test/src/test/groovy/org/zstack/test/integration/core/ansible/AnsibleRunnerTargetAwareArgumentsCase.groovy create mode 100644 test/src/test/groovy/org/zstack/test/integration/core/salt/SaltDualStackConfigCase.groovy create mode 100644 test/src/test/groovy/org/zstack/test/integration/kvm/host/DualStackAddKvmHostCase.groovy diff --git a/conf/salt/master b/conf/salt/master index 2fa75471373..1433e1ee20c 100755 --- a/conf/salt/master +++ b/conf/salt/master @@ -12,11 +12,11 @@ # # The address of the interface to bind to -#interface: 0.0.0.0 +interface: {masterInterface} # Whether the master should listen for IPv6 connections. If this is set to True, # the interface option must be adjusted too (for example: "interface: '::'") -#ipv6: False +ipv6: {ipv6Enabled} # The tcp port used by the publisher #publish_port: 4505 diff --git a/conf/salt/minion b/conf/salt/minion index 2680178055c..dd4694f5456 100755 --- a/conf/salt/minion +++ b/conf/salt/minion @@ -12,7 +12,7 @@ master: {managementNodeIp} # Set whether the minion should connect to the master via IPv6 -#ipv6: False +ipv6: {ipv6Enabled} # Set the number of seconds to wait before attempting to resolve # the master hostname if name resolution fails. Defaults to 30 seconds. diff --git a/core/src/main/java/org/zstack/core/ManagementServerIpSelection.java b/core/src/main/java/org/zstack/core/ManagementServerIpSelection.java new file mode 100644 index 00000000000..e31bd732e09 --- /dev/null +++ b/core/src/main/java/org/zstack/core/ManagementServerIpSelection.java @@ -0,0 +1,78 @@ +package org.zstack.core; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class ManagementServerIpSelection { + public enum FailureReason { + NO_SAME_FAMILY_MGT, + ROUTE_LOOKUP_FAILED, + ROUTE_SOURCE_NOT_MANAGEMENT + } + + private final String selectedIp; + private final String remoteIp; + private final int remoteFamily; + private final List managementServerIps; + private final String routeSourceIp; + private final String routeError; + private final FailureReason failureReason; + + private ManagementServerIpSelection(String selectedIp, String remoteIp, int remoteFamily, + List managementServerIps, String routeSourceIp, + String routeError, FailureReason failureReason) { + this.selectedIp = selectedIp; + this.remoteIp = remoteIp; + this.remoteFamily = remoteFamily; + this.managementServerIps = Collections.unmodifiableList(new ArrayList<>(managementServerIps)); + this.routeSourceIp = routeSourceIp; + this.routeError = routeError; + this.failureReason = failureReason; + } + + public static ManagementServerIpSelection success(String selectedIp, String remoteIp, int remoteFamily, + List managementServerIps, String routeSourceIp) { + return new ManagementServerIpSelection(selectedIp, remoteIp, remoteFamily, + managementServerIps, routeSourceIp, null, null); + } + + public static ManagementServerIpSelection failure(String remoteIp, int remoteFamily, + List managementServerIps, String routeSourceIp, + String routeError, FailureReason failureReason) { + return new ManagementServerIpSelection(null, remoteIp, remoteFamily, + managementServerIps, routeSourceIp, routeError, failureReason); + } + + public boolean isSuccess() { + return failureReason == null; + } + + public String getSelectedIp() { + return selectedIp; + } + + public String getRemoteIp() { + return remoteIp; + } + + public int getRemoteFamily() { + return remoteFamily; + } + + public List getManagementServerIps() { + return managementServerIps; + } + + public String getRouteSourceIp() { + return routeSourceIp; + } + + public String getRouteError() { + return routeError; + } + + public FailureReason getFailureReason() { + return failureReason; + } +} diff --git a/core/src/main/java/org/zstack/core/Platform.java b/core/src/main/java/org/zstack/core/Platform.java index 34df20e3e12..3363ded2b85 100755 --- a/core/src/main/java/org/zstack/core/Platform.java +++ b/core/src/main/java/org/zstack/core/Platform.java @@ -75,6 +75,9 @@ import static org.zstack.utils.CollectionDSL.e; import static org.zstack.utils.CollectionDSL.map; import static org.zstack.utils.StringDSL.ln; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002; public class Platform { private static final CLogger logger = CLoggerImpl.getLogger(Platform.class); @@ -1391,6 +1394,125 @@ public static String getManagementServerIpForRemote(String remoteIp) { return selectManagementServerIpForRemote(remoteIp, null); } + public static ManagementServerIpSelection resolveManagementServerIpForRemoteStrict(String remoteIp) { + String normalizedRemoteIp = normalizeManagementIp(remoteIp); + int remoteFamily = getIpFamily(normalizedRemoteIp); + List managementServerIps = getManagementServerIps().stream() + .map(Platform::normalizeManagementIp) + .distinct() + .collect(Collectors.toList()); + if (remoteFamily == 0) { + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, null, "remote target is not an IP address", + ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED); + } + boolean hasSameFamily = managementServerIps.stream() + .anyMatch(ip -> getIpFamily(ip) == remoteFamily); + if (!hasSameFamily) { + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, null, null, + ManagementServerIpSelection.FailureReason.NO_SAME_FAMILY_MGT); + } + + String family = remoteFamily == IPv6Constants.IPv6 ? "-6" : "-4"; + Linux.ShellResult ret = Linux.shell(String.format("/sbin/ip %s route get %s", family, normalizedRemoteIp)); + if (ret.getExitCode() != 0) { + String routeError = String.format("exitCode=%s, stdout=%s, stderr=%s", + ret.getExitCode(), ret.getStdout(), ret.getStderr()); + return selectManagementServerIpForRemoteStrict(normalizedRemoteIp, null, routeError); + } + + String routeSourceIp = parseRouteSourceIp(normalizedRemoteIp, ret.getStdout()); + if (routeSourceIp == null) { + return selectManagementServerIpForRemoteStrict(normalizedRemoteIp, null, + String.format("route output has no same-family source: %s", ret.getStdout())); + } + return selectManagementServerIpForRemoteStrict(normalizedRemoteIp, routeSourceIp, null); + } + + public static ManagementServerIpSelection selectManagementServerIpForRemoteStrict( + String remoteIp, String routeSourceIp, String routeError) { + String normalizedRemoteIp = normalizeManagementIp(remoteIp); + String normalizedRouteSourceIp = normalizeManagementIp(routeSourceIp); + int remoteFamily = getIpFamily(normalizedRemoteIp); + List managementServerIps = getManagementServerIps().stream() + .map(Platform::normalizeManagementIp) + .distinct() + .collect(Collectors.toList()); + List sameFamilyIps = managementServerIps.stream() + .filter(ip -> getIpFamily(ip) == remoteFamily) + .collect(Collectors.toList()); + + if (remoteFamily == 0) { + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, normalizedRouteSourceIp, "remote target is not an IP address", + ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED); + } + if (sameFamilyIps.isEmpty()) { + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, normalizedRouteSourceIp, routeError, + ManagementServerIpSelection.FailureReason.NO_SAME_FAMILY_MGT); + } + if (StringUtils.isBlank(normalizedRouteSourceIp)) { + String details = StringUtils.isBlank(routeError) ? "route source is unavailable" : routeError; + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, null, details, + ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED); + } + if (getIpFamily(normalizedRouteSourceIp) != remoteFamily || + !sameFamilyIps.contains(normalizedRouteSourceIp)) { + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, normalizedRouteSourceIp, routeError, + ManagementServerIpSelection.FailureReason.ROUTE_SOURCE_NOT_MANAGEMENT); + } + return ManagementServerIpSelection.success(normalizedRouteSourceIp, normalizedRemoteIp, + remoteFamily, managementServerIps, normalizedRouteSourceIp); + } + + public static ErrorCode managementServerIpSelectionError(ManagementServerIpSelection selection) { + if (selection == null || selection.isSuccess()) { + throw new IllegalArgumentException("selection must contain a failure"); + } + String context = String.format("remote[%s], family[IPv%s], managementIps%s, routeSource[%s], routeError[%s]", + selection.getRemoteIp(), selection.getRemoteFamily(), selection.getManagementServerIps(), + selection.getRouteSourceIp(), selection.getRouteError()); + switch (selection.getFailureReason()) { + case NO_SAME_FAMILY_MGT: + return operr(ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000, + "cannot find a same-family management server IP for %s", context); + case ROUTE_LOOKUP_FAILED: + return operr(ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001, + "failed to resolve the management server route source for %s", context); + case ROUTE_SOURCE_NOT_MANAGEMENT: + return operr(ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002, + "route source is not a configured management server IP for %s", context); + default: + throw new IllegalArgumentException("selection is successful or has no failure reason"); + } + } + + private static int getIpFamily(String ip) { + if (IPv6NetworkUtils.isIpv6Address(ip)) { + return IPv6Constants.IPv6; + } + if (NetworkUtils.isIpv4Address(ip)) { + return IPv6Constants.IPv4; + } + return 0; + } + + private static String parseRouteSourceIp(String remoteIp, String output) { + String[] tokens = StringUtils.defaultString(output).trim().split("\\s+"); + int remoteFamily = getIpFamily(remoteIp); + for (int i = 0; i < tokens.length - 1; i++) { + if ("src".equals(tokens[i])) { + String sourceIp = normalizeManagementIp(tokens[i + 1]); + return getIpFamily(sourceIp) == remoteFamily ? sourceIp : null; + } + } + return null; + } + public static String selectManagementServerIpForRemote(String remoteIp, String routeSourceIp) { if (StringUtils.isBlank(remoteIp)) { return getManagementServerIp(); diff --git a/core/src/main/java/org/zstack/core/agent/AgentManagerImpl.java b/core/src/main/java/org/zstack/core/agent/AgentManagerImpl.java index 27966a5c73c..1c065158027 100755 --- a/core/src/main/java/org/zstack/core/agent/AgentManagerImpl.java +++ b/core/src/main/java/org/zstack/core/agent/AgentManagerImpl.java @@ -3,6 +3,8 @@ import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.annotation.Autowired; import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.ManagementServerIpSelection; +import org.zstack.core.Platform; import org.zstack.core.ansible.AnsibleNeedRun; import org.zstack.core.ansible.AnsibleRunner; import org.zstack.core.ansible.SshFolderMd5Checker; @@ -52,6 +54,11 @@ public static String buildAgentUrl(String ip, int port, String path) { return String.format(HTTP_URL_FORMAT, IPv6NetworkUtils.formatHostForUrl(ip), port, path); } + public static String buildCommandUrl(RESTFacade restf, ManagementServerIpSelection selection) { + return selection == null ? restf.getSendCommandUrl() : + restf.buildSendCommandUrlForManagementHost(selection.getSelectedIp()); + } + @Autowired private CloudBus bus; @Autowired @@ -114,7 +121,8 @@ public String getName() { }); } - private void connect(final DeployAgentMsg msg, final Completion completion) { + private void connect(final DeployAgentMsg msg, final ManagementServerIpSelection managementServerIpSelection, + final Completion completion) { FlowChain chain = FlowChainBuilder.newShareFlowChain(); chain.setName(String.format("continue-connect-agent-server-%s:%s", msg.getIp(), msg.getAgentPort())); chain.then(new ShareFlow() { @@ -149,7 +157,8 @@ public void fail(ErrorCode errorCode) { @Override public void run(FlowTrigger trigger, Map data) { Map config = new HashMap(); - config.put(AgentConstant.CONFIG_COMMAND_URL, restf.getSendCommandUrl()); + config.put(AgentConstant.CONFIG_COMMAND_URL, + buildCommandUrl(restf, managementServerIpSelection)); if (msg.getConfig() != null) { config.putAll(msg.getConfig()); } @@ -193,6 +202,19 @@ private void deployAgent(final DeployAgentMsg msg, final NoErrorCompletion noErr return; } + final ManagementServerIpSelection managementServerIpSelection; + if (NetworkUtils.isIpAddress(msg.getIp())) { + managementServerIpSelection = Platform.resolveManagementServerIpForRemoteStrict(msg.getIp()); + if (!managementServerIpSelection.isSuccess()) { + reply.setError(Platform.managementServerIpSelectionError(managementServerIpSelection)); + bus.reply(msg, reply); + noErrorCompletion.done(); + return; + } + } else { + managementServerIpSelection = null; + } + try { String agentYamlPath = PathUtil.join(AgentConstant.ANSIBLE_MODULE_PATH, "server", AgentConstant.ANSIBLE_PLAYBOOK_NAME); File agentYaml = PathUtil.findFileOnClassPath(agentYamlPath, true); @@ -254,6 +276,7 @@ public boolean isRunNeed() { runner.setSshPort(msg.getSshPort()); } runner.setTargetIp(msg.getIp()); + runner.setManagementServerIpSelection(managementServerIpSelection); runner.setPlayBookPath(tmpAgentYaml.getAbsolutePath()); runner.run(new ReturnValueCompletion(msg, noErrorCompletion) { @Override @@ -271,7 +294,7 @@ public void run() { } }); - connect(msg, new Completion(msg, noErrorCompletion) { + connect(msg, managementServerIpSelection, new Completion(msg, noErrorCompletion) { @Override public void success() { bus.reply(msg, reply); diff --git a/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java b/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java index bd7e06ec714..ebf33492198 100755 --- a/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java +++ b/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java @@ -4,6 +4,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.Platform; +import org.zstack.core.ManagementServerIpSelection; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; import org.zstack.header.core.Completion; @@ -84,6 +86,7 @@ public class AnsibleRunner { private boolean forceRun; private AnsibleBasicArguments deployArguments; + private ManagementServerIpSelection managementServerIpSelection; public String getAnsibleExecutable() { return ansibleExecutable; @@ -197,6 +200,10 @@ public void setDeployArguments(AnsibleBasicArguments deployArguments) { this.deployArguments = deployArguments; } + public void setManagementServerIpSelection(ManagementServerIpSelection selection) { + this.managementServerIpSelection = selection; + } + public Map getArguments() { return arguments; } @@ -382,6 +389,24 @@ private void cleanup() { public void run(ReturnValueCompletion completion) { try { + String deploymentHost; + if (NetworkUtils.isIpAddress(targetIp)) { + ManagementServerIpSelection selection = managementServerIpSelection == null ? + Platform.resolveManagementServerIpForRemoteStrict(targetIp) : managementServerIpSelection; + if (!selection.isSuccess()) { + completion.fail(Platform.managementServerIpSelectionError(selection)); + return; + } + deploymentHost = selection.getSelectedIp(); + } else { + deploymentHost = restf.getHostName(); + } + for (AnsibleChecker checker : checkers) { + if (checker instanceof ManagementServerIpAwareChecker) { + ((ManagementServerIpAwareChecker) checker).setManagementServerIp(deploymentHost); + } + } + if (!forceRun && !isNeedRun()) { completion.success(false); return; @@ -393,9 +418,9 @@ public void run(ReturnValueCompletion completion) { deployArguments = new AnsibleBasicArguments(); } - deployArguments.setPipUrl(buildPipUrl(restf.getHostName(), port)); - deployArguments.setTrustedHost(restf.getHostName()); - deployArguments.setYumServer(IPv6NetworkUtils.formatHostPort(restf.getHostName(), port)); + deployArguments.setPipUrl(buildPipUrl(deploymentHost, port)); + deployArguments.setTrustedHost(deploymentHost); + deployArguments.setYumServer(IPv6NetworkUtils.formatHostPort(deploymentHost, port)); deployArguments.setRemoteUser(username); if (password != null && !password.isEmpty()) { deployArguments.setRemotePass(password); diff --git a/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java b/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java index 5cf5665c06b..63720a51151 100644 --- a/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java @@ -19,7 +19,7 @@ /** * Created by mingjian.deng on 2019/6/12. */ -public class CallBackNetworkChecker implements AnsibleChecker { +public class CallBackNetworkChecker implements AnsibleChecker, ManagementServerIpAwareChecker { private static final CLogger logger = Utils.getLogger(CallBackNetworkChecker.class); private String username; @@ -136,6 +136,11 @@ public void setCallbackIp(String callbackIp) { this.callbackIp = callbackIp; } + @Override + public void setManagementServerIp(String managementServerIp) { + setCallbackIp(managementServerIp); + } + public int getCallBackPort() { return callBackPort; } diff --git a/core/src/main/java/org/zstack/core/ansible/ManagementServerIpAwareChecker.java b/core/src/main/java/org/zstack/core/ansible/ManagementServerIpAwareChecker.java new file mode 100644 index 00000000000..4d91567f8f9 --- /dev/null +++ b/core/src/main/java/org/zstack/core/ansible/ManagementServerIpAwareChecker.java @@ -0,0 +1,5 @@ +package org.zstack.core.ansible; + +public interface ManagementServerIpAwareChecker { + void setManagementServerIp(String managementServerIp); +} diff --git a/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java b/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java index 8438fce4eac..c446df48a77 100644 --- a/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java @@ -7,22 +7,33 @@ import org.zstack.header.rest.RESTFacade; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; +import org.zstack.utils.network.IPv6NetworkUtils; import org.zstack.utils.ssh.Ssh; import org.zstack.utils.ssh.SshResult; +import java.net.URI; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + /** * Created by GuoYi on 2018-12-24. */ @Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) -public class SshYumRepoChecker implements AnsibleChecker { +public class SshYumRepoChecker implements AnsibleChecker, ManagementServerIpAwareChecker { @Autowired protected RESTFacade restf; private static final CLogger logger = Utils.getLogger(SshYumRepoChecker.class); + private static final String ZSTACK_REPO = "/etc/yum.repos.d/zstack-mn.repo"; + private static final String QEMU_REPO = "/etc/yum.repos.d/qemu-kvm-ev-mn.repo"; + private static final String READ_REPO_BASEURLS = String.format( + "grep -H -E '^[[:space:]]*baseurl[[:space:]]*=' %s %s", ZSTACK_REPO, QEMU_REPO); private String username; private String password; private String privateKey; private String targetIp; + private String managementServerIp; private int sshPort = 22; @Override @@ -36,9 +47,7 @@ public boolean needDeploy() { .setPassword(password).setPort(sshPort) .setHostname(targetIp); try { - ssh.sudoCommand(String.format("sed -i '/baseurl/s/\\([0-9]\\{1,3\\}\\.\\)\\{3\\}[0-9]\\{1,3\\}:\\([0-9]\\+\\)/%s/g' /etc/yum.repos.d/{zstack,qemu-kvm-ev}-mn.repo", - restf.getHostName() + ":" + restf.getPort() - )); + ssh.sudoCommand(repoReadCommand()); SshResult ret = ssh.setTimeout(60).runAndClose(); if (ret.getReturnCode() != 0) { logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s", @@ -46,15 +55,71 @@ public boolean needDeploy() { return true; } - ssh.reset(); + if (StringUtils.isBlank(managementServerIp)) { + logger.warn(String.format("management server IP is not set for repo checker on target[%s]", targetIp)); + return true; + } + if (!repoOutputUsesEndpoint(ret.getStdout(), managementServerIp, restf.getPort())) { + logger.debug(String.format("yum repo endpoint on target[%s] does not match management node[%s:%s]", + targetIp, managementServerIp, restf.getPort())); + return true; + } + } catch (Exception e) { + logger.warn(String.format("failed to check yum repo endpoint on target[%s]", targetIp), e); + return true; } finally { ssh.close(); } - logger.debug("successfully configured zstack-mn.repo and qemu-kvm-ev-mn.repo in " + targetIp); + logger.debug("yum repo endpoints match the management node on target " + targetIp); return false; } + static boolean repoOutputUsesEndpoint(String output, String expectedHost, int expectedPort) { + if (StringUtils.isBlank(output) || StringUtils.isBlank(expectedHost)) { + return false; + } + + Set expectedRepos = new HashSet<>(Arrays.asList(ZSTACK_REPO, QEMU_REPO)); + Set matchedRepos = new HashSet<>(); + for (String line : output.split("\\r?\\n")) { + int pathSeparator = line.indexOf(':'); + int valueSeparator = line.indexOf('=', pathSeparator + 1); + if (pathSeparator <= 0 || valueSeparator <= pathSeparator || + !"baseurl".equals(line.substring(pathSeparator + 1, valueSeparator).trim())) { + return false; + } + + String repo = line.substring(0, pathSeparator).trim(); + if (!expectedRepos.contains(repo)) { + return false; + } + + try { + URI baseUrl = new URI(line.substring(valueSeparator + 1).trim()); + String actualHost = IPv6NetworkUtils.stripHostUrlBrackets(baseUrl.getHost()); + String normalizedExpectedHost = IPv6NetworkUtils.isIpv6Address(expectedHost) ? + IPv6NetworkUtils.normalizeIpv6(expectedHost) : expectedHost; + String normalizedActualHost = IPv6NetworkUtils.isIpv6Address(actualHost) ? + IPv6NetworkUtils.normalizeIpv6(actualHost) : actualHost; + if (normalizedActualHost == null || + !normalizedExpectedHost.equalsIgnoreCase(normalizedActualHost) || + baseUrl.getPort() != expectedPort) { + return false; + } + } catch (Exception e) { + return false; + } + matchedRepos.add(repo); + } + + return matchedRepos.containsAll(expectedRepos); + } + + static String repoReadCommand() { + return READ_REPO_BASEURLS; + } + @Override public void deleteDestFile() { @@ -99,4 +164,9 @@ public String getTargetIp() { public void setTargetIp(String targetIp) { this.targetIp = targetIp; } + + @Override + public void setManagementServerIp(String managementServerIp) { + this.managementServerIp = managementServerIp; + } } diff --git a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java index 26dbb7ec675..e3215077f79 100755 --- a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java @@ -62,6 +62,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import static org.zstack.core.Platform.*; import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; @@ -193,7 +194,8 @@ void init() { port = Platform.getManagementNodeServicePort(); - IptablesUtils.insertRuleToFilterTable(String.format("-A INPUT -p tcp -m state --state NEW -m tcp --dport %s -j ACCEPT", port)); + installRestPortFirewallRules(port, Platform.getManagementServerIp6() != null, + IptablesUtils::insertRuleToFilterTable, IptablesUtils::insertRuleToFilterTableV6); if ("AUTO".equals(hostname)) { callbackHostName = Platform.getManagementServerIp(); @@ -215,6 +217,16 @@ void init() { CoreGlobalProperty.REST_FACADE_MAX_TOTAL); } + public static void installRestPortFirewallRules(int port, boolean ipv6Enabled, + Consumer ipv4RuleInstaller, + Consumer ipv6RuleInstaller) { + String rule = String.format("-A INPUT -p tcp -m state --state NEW -m tcp --dport %s -j ACCEPT", port); + ipv4RuleInstaller.accept(rule); + if (ipv6Enabled) { + ipv6RuleInstaller.accept(rule); + } + } + public static String buildBaseUrl(String hostName, int port, String path) { String url = IPv6NetworkUtils.buildHttpUrl(hostName, port); if (path != null && !path.isEmpty()) { @@ -260,6 +272,13 @@ public static String buildSendCommandUrl(String hostName, int port, String path) return ub.build().toUriString(); } + public static String selectSendCommandUrl(String remoteHost, String defaultSendCommandUrl, int port, String path) { + if (!NetworkUtils.isIpAddress(remoteHost)) { + return defaultSendCommandUrl; + } + return buildSendCommandUrl(Platform.getManagementServerIpForRemote(remoteHost), port, path); + } + // cap the agent-advertised keep-alive to keepAliveMs; also cap when the agent sends none (duration < 0) static long cappedKeepAlive(long serverDuration, long capMs) { return (serverDuration < 0 || serverDuration > capMs) ? capMs : serverDuration; @@ -1061,6 +1080,16 @@ public String getSendCommandUrl() { return sendCommandUrl; } + @Override + public String buildSendCommandUrl(String remoteHost) { + return selectSendCommandUrl(remoteHost, sendCommandUrl, port, path); + } + + @Override + public String buildSendCommandUrlForManagementHost(String managementHost) { + return buildSendCommandUrl(managementHost, port, path); + } + @Override public String getCallbackUrl() { return callbackUrl; diff --git a/core/src/main/java/org/zstack/core/salt/SaltFacadeImpl.java b/core/src/main/java/org/zstack/core/salt/SaltFacadeImpl.java index 7f522167ca9..d1a21e1d9ae 100755 --- a/core/src/main/java/org/zstack/core/salt/SaltFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/salt/SaltFacadeImpl.java @@ -6,6 +6,7 @@ import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.Platform; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.MessageSafe; import org.zstack.header.AbstractService; @@ -64,19 +65,25 @@ private boolean deployFile(File src, File dst) throws IOException { private File rewriteMasterConfFile() throws IOException { File masterConfTmpt = PathUtil.findFileOnClassPath(PathUtil.join("salt", SaltConstant.MASTER_CONF_NAME), true); - - Map map = new HashMap(); - map.put("maxOpenFiles", String.valueOf(masterMaxOpenFiles)); - map.put("workerThreads", String.valueOf(masterWorkerThreads)); - String srcConf = FileUtils.readFileToString(masterConfTmpt); - String conf = StringTemplate.substitute(srcConf, map); + boolean ipv6Enabled = Platform.getManagementServerIp6() != null; + String conf = renderMasterConfig(srcConf, masterMaxOpenFiles, masterWorkerThreads, ipv6Enabled); File masterConf = File.createTempFile("zstack-salt", "master"); FileUtils.write(masterConf, conf); return masterConf; } + public static String renderMasterConfig(String template, int maxOpenFiles, int workerThreads, + boolean ipv6Enabled) { + Map map = new HashMap(); + map.put("maxOpenFiles", String.valueOf(maxOpenFiles)); + map.put("workerThreads", String.valueOf(workerThreads)); + map.put("masterInterface", ipv6Enabled ? "'::'" : "0.0.0.0"); + map.put("ipv6Enabled", String.valueOf(ipv6Enabled)); + return StringTemplate.substitute(template, map); + } + private void prepareSaltMaster() throws IOException { @@ -84,6 +91,10 @@ private void prepareSaltMaster() throws IOException { try { IptablesUtils.insertRuleToFilterTable("-A INPUT -p tcp -m state --state NEW -m tcp --dport 4505 -j ACCEPT"); IptablesUtils.insertRuleToFilterTable("-A INPUT -p tcp -m state --state NEW -m tcp --dport 4506 -j ACCEPT"); + if (Platform.getManagementServerIp6() != null) { + IptablesUtils.insertRuleToFilterTableV6("-A INPUT -p tcp -m state --state NEW -m tcp --dport 4505 -j ACCEPT"); + IptablesUtils.insertRuleToFilterTableV6("-A INPUT -p tcp -m state --state NEW -m tcp --dport 4506 -j ACCEPT"); + } File dstMasterConf = new File(PathUtil.join(SaltConstant.SALT_CONF_HOME, SaltConstant.MASTER_CONF_NAME)); if (deployFile(srcMasterConf, dstMasterConf)) { @@ -322,4 +333,3 @@ public void setMasterFileRoots(String masterFileRoots) { this.masterFileRoots = masterFileRoots; } } - diff --git a/core/src/main/java/org/zstack/core/salt/SaltSetupMinionJob.java b/core/src/main/java/org/zstack/core/salt/SaltSetupMinionJob.java index f2b40a50ad2..efeeaace25c 100755 --- a/core/src/main/java/org/zstack/core/salt/SaltSetupMinionJob.java +++ b/core/src/main/java/org/zstack/core/salt/SaltSetupMinionJob.java @@ -6,6 +6,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.Platform; +import org.zstack.core.ManagementServerIpSelection; import org.zstack.core.errorcode.ErrorFacade; import org.zstack.core.job.Job; import org.zstack.core.job.JobContext; @@ -13,6 +14,8 @@ import org.zstack.utils.Utils; import org.zstack.utils.data.StringTemplate; import org.zstack.utils.logging.CLogger; +import org.zstack.utils.network.IPv6NetworkUtils; +import org.zstack.utils.network.NetworkUtils; import org.zstack.utils.path.PathUtil; import org.zstack.utils.ssh.Ssh; import org.zstack.utils.ssh.SshException; @@ -52,32 +55,48 @@ public class SaltSetupMinionJob implements Job { private String minionId; @JobContext private boolean cleanMasterKey; + private ManagementServerIpSelection managementServerIpSelection; @Autowired private ErrorFacade errf; private static final String SALT_BOOTSTRAP = "salt/salt-bootstrap.sh"; - private File rewriteMinionConfFile(String minionId) throws IOException { + private File rewriteMinionConfFile(String minionId, String managementNodeIp) throws IOException { File minionConfTmpt = new File(saltMinionConfPath); - - Map map = new HashMap(); - map.put("managementNodeIp", Platform.getManagementServerIp()); - map.put("minionId", minionId); - String srcConf = FileUtils.readFileToString(minionConfTmpt); - String conf = StringTemplate.substitute(srcConf, map); + String conf = renderMinionConfig(srcConf, managementNodeIp, minionId); File minionConf = File.createTempFile("zstack-salt", "minion"); FileUtils.write(minionConf, conf); return minionConf; } + public static String renderMinionConfig(String template, String managementNodeIp, String minionId) { + Map map = new HashMap(); + map.put("managementNodeIp", managementNodeIp); + map.put("ipv6Enabled", String.valueOf(IPv6NetworkUtils.isIpv6Address(managementNodeIp))); + map.put("minionId", minionId); + return StringTemplate.substitute(template, map); + } + @Override public void run(ReturnValueCompletion completion) { File tmpt = null; Ssh ssh = null; FileInputStream fis = null; try { + String managementNodeIp; + if (NetworkUtils.isIpAddress(targetIp)) { + ManagementServerIpSelection selection = managementServerIpSelection == null ? + Platform.resolveManagementServerIpForRemoteStrict(targetIp) : managementServerIpSelection; + if (!selection.isSuccess()) { + completion.fail(Platform.managementServerIpSelectionError(selection)); + return; + } + managementNodeIp = selection.getSelectedIp(); + } else { + managementNodeIp = Platform.getManagementServerIp(); + } ssh = new Ssh().setHostname(targetIp).setPassword(password).setPrivateKey(privateKey) .setUsername(username).setPort(port); SshResult ret = ssh.checkTool("scp").run(); @@ -100,7 +119,7 @@ public void run(ReturnValueCompletion completion) { logger.debug(String.format("salt-minion is found on system[%s], no need to install new one", targetIp)); } - tmpt = rewriteMinionConfFile(minionId); + tmpt = rewriteMinionConfFile(minionId, managementNodeIp); String minionConfPath = PathUtil.join(SaltConstant.SALT_CONF_HOME, SaltConstant.MINION_CONF_NAME); boolean deployMinion = false; ret = ssh.reset().command(String.format("md5sum %s", minionConfPath)).run(); @@ -167,6 +186,10 @@ public void setTargetIp(String targetIp) { this.targetIp = targetIp; } + public void setManagementServerIpSelection(ManagementServerIpSelection selection) { + this.managementServerIpSelection = selection; + } + public String getUsername() { return username; } diff --git a/header/src/main/java/org/zstack/header/rest/RESTFacade.java b/header/src/main/java/org/zstack/header/rest/RESTFacade.java index 23fee5208c2..0dd4786fc7c 100755 --- a/header/src/main/java/org/zstack/header/rest/RESTFacade.java +++ b/header/src/main/java/org/zstack/header/rest/RESTFacade.java @@ -89,6 +89,10 @@ public interface RESTFacade { String getSendCommandUrl(); + String buildSendCommandUrl(String remoteHost); + + String buildSendCommandUrlForManagementHost(String managementHost); + String getCallbackUrl(); /** diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 226ebbfbcaa..ff84463eb14 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -20,6 +20,7 @@ import org.zstack.header.vm.devices.VmInstanceDeviceManager; import org.zstack.core.CoreGlobalProperty; import org.zstack.core.MessageCommandRecorder; +import org.zstack.core.ManagementServerIpSelection; import org.zstack.core.Platform; import org.zstack.core.agent.AgentConstant; import org.zstack.core.ansible.*; @@ -74,6 +75,7 @@ import org.zstack.header.network.l3.L3NetworkInventory; import org.zstack.header.network.l3.L3NetworkVO; import org.zstack.header.rest.JsonAsyncRESTCallback; +import org.zstack.header.rest.RESTConstant; import org.zstack.header.rest.RESTFacade; import org.zstack.header.storage.primary.*; import org.zstack.header.storage.snapshot.*; @@ -133,6 +135,7 @@ import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; public class KVMHost extends HostBase implements Host { + static final String UNIT_TEST_AGENT_HOST = "KVMHost.unitTestAgentHost"; private static final CLogger logger = Utils.getLogger(KVMHost.class); private static final ZTester tester = Utils.getTester(); private static final String EXTRA_IP_SEPARATOR = ","; @@ -216,6 +219,7 @@ public static Set parseSanIps(String sanOutput) { protected KVMHostFactory factory; @Autowired private RESTFacade restf; + private volatile ManagementServerIpSelection managementServerIpSelection; @Autowired private KVMExtensionEmitter extEmitter; @Autowired @@ -318,7 +322,8 @@ public KVMHost(KVMHostVO self, KVMHostContext context) { super(self); this.context = context; - baseUrl = context.getBaseUrl(); + baseUrl = isUnitTestAgentRedirectEnabled() ? + buildAgentUrl(System.getProperty(UNIT_TEST_AGENT_HOST), "") : context.getBaseUrl(); UriComponentsBuilder ub = UriComponentsBuilder.fromHttpUrl(baseUrl); ub.path(KVMConstant.KVM_CONNECT_PATH); @@ -559,6 +564,7 @@ class Http { Class responseClass; String commandStr; String commandName; + String targetManagementIp; long timeout = -1; Http timeout(long timeout) { @@ -566,6 +572,11 @@ Http timeout(long timeout) { return this; } + Http targetManagementIp(String targetManagementIp) { + this.targetManagementIp = targetManagementIp; + return this; + } + public Http(String path, String cmd, String commandName, Class rspClz) { this.path = path; this.commandStr = cmd; @@ -593,6 +604,17 @@ void call(String resourceUuid, ReturnValueCompletion completion) { Map header = new HashMap<>(); header.put(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID, resourceUuid == null ? self.getUuid() : resourceUuid); + String callbackTarget = targetManagementIp == null ? self.getManagementIp() : targetManagementIp; + if (NetworkUtils.isIpAddress(callbackTarget)) { + ManagementServerIpSelection selection = + Platform.resolveManagementServerIpForRemoteStrict(callbackTarget); + if (!selection.isSuccess()) { + completion.fail(Platform.managementServerIpSelectionError(selection)); + return; + } + header.put(RESTConstant.CALLBACK_URL, + restf.buildCallbackUrl(selection.getSelectedIp())); + } runBeforeAsyncJsonPostExts(header); if (commandStr != null) { long overall = timeoutManager.getTimeout(); @@ -2903,9 +2925,9 @@ public static String buildManagementNodeCallbackCheckCommand(String callbackUrl) return String.format(MANAGEMENT_NODE_CALLBACK_CHECK_COMMAND, callbackUrl, callbackUrl); } - public static String buildManagementNodeCallbackCheckCommand(String hostManagementIp, RESTFacade restf) { - String callbackHost = Platform.getManagementServerIpForRemote(hostManagementIp); - return buildManagementNodeCallbackCheckCommand(restf.buildCallbackUrl(callbackHost)); + public static String buildManagementNodeCallbackCheckCommandForManagementHost( + String managementHost, RESTFacade restf) { + return buildManagementNodeCallbackCheckCommand(restf.buildCallbackUrl(managementHost)); } private static String joinAgentPath(String rootPath, String path) { @@ -2927,7 +2949,14 @@ private static String joinAgentPath(String rootPath, String path) { } private String buildUrl(String path) { - return buildAgentUrl(self.getManagementIp(), path); + String host = isUnitTestAgentRedirectEnabled() ? + System.getProperty(UNIT_TEST_AGENT_HOST) : self.getManagementIp(); + return buildAgentUrl(host, path); + } + + private static boolean isUnitTestAgentRedirectEnabled() { + return CoreGlobalProperty.UNIT_TEST_ON && + StringUtils.isNotBlank(System.getProperty(UNIT_TEST_AGENT_HOST)); } private void executeAsyncHttpCall(final KVMHostAsyncHttpCallMsg msg, final NoErrorCompletion completion) { @@ -3370,7 +3399,8 @@ public void run(final FlowTrigger trigger, Map data) { cmd.vmUuid = vmUuid; String url = buildAgentUrl(dstHostMnIp, KVMConstant.CLEAN_FIRMWARE_FLASH); - new Http<>(url, cmd, AgentResponse.class).call(dstHostUuid, new ReturnValueCompletion(trigger) { + new Http<>(url, cmd, AgentResponse.class).targetManagementIp(dstHostMnIp) + .call(dstHostUuid, new ReturnValueCompletion(trigger) { @Override public void success(AgentResponse ret) { if (!ret.isSuccess()) { @@ -3442,7 +3472,9 @@ protected void scripts() { String migrateUrl = buildAgentUrl( migrateFromDestination ? dstHostMnIp : srcHostMnIp, KVMConstant.KVM_MIGRATE_VM_PATH); - new Http<>(migrateUrl, cmd, MigrateVmResponse.class).call(migrateFromDestination ? dstHostUuid : srcHostUuid, new ReturnValueCompletion(trigger) { + new Http<>(migrateUrl, cmd, MigrateVmResponse.class) + .targetManagementIp(migrateFromDestination ? dstHostMnIp : srcHostMnIp) + .call(migrateFromDestination ? dstHostUuid : srcHostUuid, new ReturnValueCompletion(trigger) { @Override public void success(MigrateVmResponse ret) { if (!ret.isSuccess()) { @@ -3481,7 +3513,8 @@ public void run(final FlowTrigger trigger, Map data) { cmd.hostManagementIp = dstHostMnIp; String url = buildAgentUrl(dstHostMnIp, KVMConstant.KVM_HARDEN_CONSOLE_PATH); - new Http<>(url, cmd, AgentResponse.class).call(dstHostUuid, new ReturnValueCompletion(trigger) { + new Http<>(url, cmd, AgentResponse.class).targetManagementIp(dstHostMnIp) + .call(dstHostUuid, new ReturnValueCompletion(trigger) { @Override public void success(AgentResponse ret) { if (!ret.isSuccess()) { @@ -3514,7 +3547,8 @@ public void run(final FlowTrigger trigger, Map data) { cmd.hostManagementIp = srcHostMnIp; String url = buildAgentUrl(srcHostMnIp, KVMConstant.KVM_DELETE_CONSOLE_FIREWALL_PATH); - new Http<>(url, cmd, AgentResponse.class).call(new ReturnValueCompletion(trigger) { + new Http<>(url, cmd, AgentResponse.class).targetManagementIp(srcHostMnIp) + .call(new ReturnValueCompletion(trigger) { @Override public void success(AgentResponse ret) { if (!ret.isSuccess()) { @@ -5177,7 +5211,7 @@ private void doReconnectHostDueToPingResult(PingResponse ret) { bus.send(rmsg); } - private void doUpdateHostConfiguration() { + private void doUpdateHostConfiguration(ManagementServerIpSelection pingSelection) { thdf.chainSubmit(new ChainTask(null) { @Override public String getSyncSignature() { @@ -5186,9 +5220,29 @@ public String getSyncSignature() { @Override public void run(SyncTaskChain chain) { + ManagementServerIpSelection selection = pingSelection; + if (NetworkUtils.isIpAddress(self.getManagementIp())) { + if (selection == null) { + selection = Platform.resolveManagementServerIpForRemoteStrict(self.getManagementIp()); + } + if (!selection.isSuccess()) { + ErrorCode error = Platform.managementServerIpSelectionError(selection); + logger.warn(String.format("failed to select management server IP for host[uuid:%s]: %s", + self.getUuid(), error)); + changeConnectionState(HostStatusEvent.disconnected); + new HostDisconnectedCanonicalEvent(self.getUuid(), error).fire(); + ReconnectHostMsg rmsg = new ReconnectHostMsg(); + rmsg.setHostUuid(self.getUuid()); + bus.makeTargetServiceIdByResourceUuid(rmsg, HostConstant.SERVICE_ID, self.getUuid()); + bus.send(rmsg); + chain.next(); + return; + } + } UpdateHostConfigurationCmd cmd = new UpdateHostConfigurationCmd(); cmd.hostUuid = self.getUuid(); - cmd.sendCommandUrl = restf.getSendCommandUrl(); + cmd.sendCommandUrl = selection == null ? restf.buildSendCommandUrl(self.getManagementIp()) : + restf.buildSendCommandUrlForManagementHost(selection.getSelectedIp()); restf.asyncJsonPost(updateHostConfigurationPath, cmd, new JsonAsyncRESTCallback(chain) { @Override public void fail(ErrorCode err) { @@ -5267,9 +5321,26 @@ protected void connectHostFailHook(ErrorCode errorCode) { super.connectHostFailHook(errorCode); } - boolean needUpdateHostConfiguration(PingResponse rsp) { + boolean needUpdateHostConfiguration(PingResponse rsp, ManagementServerIpSelection selection) { // host uuid or send command url or version changed - return !restf.getSendCommandUrl().equals(rsp.getSendCommandUrl()); + if (NetworkUtils.isIpAddress(self.getManagementIp())) { + if (selection == null) { + selection = Platform.resolveManagementServerIpForRemoteStrict(self.getManagementIp()); + } + if (!selection.isSuccess()) { + if (selection.getFailureReason() == + ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED) { + logger.warn(String.format("skip updating host[uuid:%s] configuration because route lookup " + + "for management IP[%s] failed: %s", self.getUuid(), self.getManagementIp(), + selection.getRouteError())); + return false; + } + return true; + } + } + String sendCommandUrl = selection == null ? restf.buildSendCommandUrl(self.getManagementIp()) : + restf.buildSendCommandUrlForManagementHost(selection.getSelectedIp()); + return !sendCommandUrl.equals(rsp.getSendCommandUrl()); } @Override @@ -5316,8 +5387,14 @@ public void success(PingResponse ret) { } afterDone.add(() -> doReconnectHostDueToPingResult(ret)); - } else if (needUpdateHostConfiguration(ret)) { - afterDone.add(KVMHost.this::doUpdateHostConfiguration); + } else { + ManagementServerIpSelection pingSelection = + NetworkUtils.isIpAddress(self.getManagementIp()) ? + Platform.resolveManagementServerIpForRemoteStrict( + self.getManagementIp()) : null; + if (needUpdateHostConfiguration(ret, pingSelection)) { + afterDone.add(() -> doUpdateHostConfiguration(pingSelection)); + } } trigger.next(); @@ -5448,9 +5525,18 @@ protected int getVmMigrateQuantity() { private ErrorCode connectToAgent() { ErrorCode errCode = null; try { + ManagementServerIpSelection selection = null; + if (NetworkUtils.isIpAddress(self.getManagementIp())) { + selection = Platform.resolveManagementServerIpForRemoteStrict(self.getManagementIp()); + if (!selection.isSuccess()) { + return Platform.managementServerIpSelectionError(selection); + } + managementServerIpSelection = selection; + } ConnectCmd cmd = new ConnectCmd(); cmd.setHostUuid(self.getUuid()); - cmd.setSendCommandUrl(restf.getSendCommandUrl()); + cmd.setSendCommandUrl(selection == null ? restf.buildSendCommandUrl(self.getManagementIp()) : + restf.buildSendCommandUrlForManagementHost(selection.getSelectedIp())); cmd.setIptablesRules(KVMGlobalProperty.IPTABLES_RULES); cmd.setIgnoreMsrs(KVMGlobalConfig.KVM_IGNORE_MSRS.value(Boolean.class)); cmd.setTcpServerPort(KVMGlobalProperty.TCP_SERVER_PORT); @@ -5808,8 +5894,18 @@ public void run(FlowTrigger trigger, Map data) { sshShell.setPassword(getSelf().getPassword()); sshShell.setPort(getSelf().getPort()); sshShell.setWithSudo(false); - final String callbackHost = Platform.getManagementServerIpForRemote(getSelf().getManagementIp()); - final String cmd = buildManagementNodeCallbackCheckCommand(getSelf().getManagementIp(), restf); + String callbackHost = restf.getHostName(); + if (NetworkUtils.isIpAddress(getSelf().getManagementIp())) { + ManagementServerIpSelection selection = + Platform.resolveManagementServerIpForRemoteStrict(getSelf().getManagementIp()); + if (!selection.isSuccess()) { + trigger.fail(Platform.managementServerIpSelectionError(selection)); + return; + } + managementServerIpSelection = selection; + callbackHost = selection.getSelectedIp(); + } + final String cmd = buildManagementNodeCallbackCheckCommandForManagementHost(callbackHost, restf); SshResult ret = sshShell.runCommand(cmd); if (ret.getStderr() != null && ret.getStderr().contains("No route to host")) { // c.f. https://access.redhat.com/solutions/1120533 @@ -6058,12 +6154,25 @@ public void run(FlowTrigger trigger, Map data) { @Override public boolean skip(Map data) { - return CoreGlobalProperty.UNIT_TEST_ON; + return CoreGlobalProperty.UNIT_TEST_ON && !isUnitTestAgentRedirectEnabled(); } @Override public void run(final FlowTrigger trigger, Map data) { - String srcPath = PathUtil.findFileOnClassPath(String.format("ansible/kvm/%s", agentPackageName), true).getAbsolutePath(); + ManagementServerIpSelection selection = null; + String selectedManagementServerIp = restf.getHostName(); + if (NetworkUtils.isIpAddress(getSelf().getManagementIp())) { + selection = Platform.resolveManagementServerIpForRemoteStrict( + getSelf().getManagementIp()); + if (!selection.isSuccess()) { + trigger.fail(Platform.managementServerIpSelectionError(selection)); + return; + } + managementServerIpSelection = selection; + selectedManagementServerIp = selection.getSelectedIp(); + } + String srcPath = isUnitTestAgentRedirectEnabled() ? agentPackageName : + PathUtil.findFileOnClassPath(String.format("ansible/kvm/%s", agentPackageName), true).getAbsolutePath(); String destPath = String.format("/var/lib/zstack/kvm/package/%s", agentPackageName); SshFileMd5Checker checker = new SshFileMd5Checker(); checker.setUsername(getSelf().getUsername()); @@ -6097,7 +6206,7 @@ public void run(final FlowTrigger trigger, Map data) { callbackChecker.setUsername(getSelf().getUsername()); callbackChecker.setPassword(getSelf().getPassword()); callbackChecker.setPort(getSelf().getPort()); - callbackChecker.setCallbackIp(Platform.getManagementServerIpForRemote(getSelf().getManagementIp())); + callbackChecker.setCallbackIp(selectedManagementServerIp); callbackChecker.setCallBackPort(CloudBusGlobalProperty.HTTP_PORT); KvmHostConfigChecker kvmHostConfigChecker = new KvmHostConfigChecker(); @@ -6121,7 +6230,7 @@ public void run(final FlowTrigger trigger, Map data) { hostTcpConnectionCallbackChecker.setUsername(getSelf().getUsername()); hostTcpConnectionCallbackChecker.setPassword(getSelf().getPassword()); hostTcpConnectionCallbackChecker.setPort(getSelf().getPort()); - hostTcpConnectionCallbackChecker.setCallbackIp(Platform.getManagementServerIpForRemote(getSelf().getManagementIp())); + hostTcpConnectionCallbackChecker.setCallbackIp(selectedManagementServerIp); hostTcpConnectionCallbackChecker.setCallBackPort(KVMGlobalProperty.TCP_SERVER_PORT); runner.installChecker(hostTcpConnectionCallbackChecker); } @@ -6133,6 +6242,7 @@ public void run(final FlowTrigger trigger, Map data) { } } runner.setAgentPort(KVMGlobalProperty.AGENT_PORT); + runner.setManagementServerIpSelection(selection); runner.setTargetIp(getSelf().getManagementIp()); runner.setTargetUuid(getSelf().getUuid()); runner.setPlayBookName(KVMConstant.ANSIBLE_PLAYBOOK_NAME); diff --git a/test/src/test/groovy/org/zstack/core/agent/AgentManagerCommandUrlCase.groovy b/test/src/test/groovy/org/zstack/core/agent/AgentManagerCommandUrlCase.groovy new file mode 100644 index 00000000000..ba3ea2c009f --- /dev/null +++ b/test/src/test/groovy/org/zstack/core/agent/AgentManagerCommandUrlCase.groovy @@ -0,0 +1,30 @@ +package org.zstack.core.agent + +import org.junit.Test +import org.zstack.core.ManagementServerIpSelection +import org.zstack.core.rest.RESTFacadeImpl +import org.zstack.header.rest.RESTFacade + +class AgentManagerCommandUrlCase { + private static final String IPV4 = "192.168.1.10" + private static final String IPV6 = "2001:db8::1" + private static final String REMOTE_IPV6 = "2001:db8::2" + private static final int REST_PORT = 8080 + + @Test + void testStrictSelectionAndHostnameFallback() { + RESTFacade restf = [ + getSendCommandUrl: { "http://${IPV4}:${REST_PORT}/zstack/command".toString() }, + buildSendCommandUrlForManagementHost: { String host -> + RESTFacadeImpl.buildSendCommandUrl(host, REST_PORT, "/zstack") + } + ] as RESTFacade + ManagementServerIpSelection selection = ManagementServerIpSelection.success( + IPV6, REMOTE_IPV6, 6, [IPV4, IPV6], IPV6) + + assert AgentManagerImpl.buildCommandUrl(restf, selection) == + "http://[${IPV6}]:${REST_PORT}/zstack/asyncrest/sendcommand" + assert AgentManagerImpl.buildCommandUrl(restf, null) == + "http://${IPV4}:${REST_PORT}/zstack/command" + } +} diff --git a/test/src/test/groovy/org/zstack/core/ansible/SshYumRepoCheckerCase.groovy b/test/src/test/groovy/org/zstack/core/ansible/SshYumRepoCheckerCase.groovy new file mode 100644 index 00000000000..ee466df2b2f --- /dev/null +++ b/test/src/test/groovy/org/zstack/core/ansible/SshYumRepoCheckerCase.groovy @@ -0,0 +1,49 @@ +package org.zstack.core.ansible + +import org.junit.Test + +class SshYumRepoCheckerCase { + private static final String ZSTACK_REPO = "/etc/yum.repos.d/zstack-mn.repo" + private static final String QEMU_REPO = "/etc/yum.repos.d/qemu-kvm-ev-mn.repo" + + @Test + void testRepoEndpoints() { + assert SshYumRepoChecker.repoReadCommand().startsWith("grep ") + assert !SshYumRepoChecker.repoReadCommand().contains("sed -i") + + assert SshYumRepoChecker.repoOutputUsesEndpoint(repoOutput( + "http://[2001:db8::10]:8080/zstack/static/zstack-repo/x86_64", + "http://[2001:db8::10]:8080/zstack/static/qemu-kvm-ev/x86_64"), + "2001:db8::10", 8080) + + assert SshYumRepoChecker.repoOutputUsesEndpoint(repoOutput( + "http://192.168.1.10:8080/zstack/static/zstack-repo/x86_64", + "http://192.168.1.10:8080/zstack/static/qemu-kvm-ev/x86_64"), + "192.168.1.10", 8080) + + assert SshYumRepoChecker.repoOutputUsesEndpoint(repoOutput( + "http://mn.example.com:8080/zstack/static/zstack-repo/x86_64", + "http://mn.example.com:8080/zstack/static/qemu-kvm-ev/x86_64"), + "mn.example.com", 8080) + } + + @Test + void testMismatchMissingAndMalformedRepoEndpoints() { + assert !SshYumRepoChecker.repoOutputUsesEndpoint(repoOutput( + "http://[2001:db8::10]:8080/zstack/static/zstack-repo/x86_64", + "http://192.168.1.10:8080/zstack/static/qemu-kvm-ev/x86_64"), + "2001:db8::10", 8080) + + assert !SshYumRepoChecker.repoOutputUsesEndpoint( + "${ZSTACK_REPO}:baseurl=http://[2001:db8::10]:8080/zstack/static/zstack-repo/x86_64\n", + "2001:db8::10", 8080) + + assert !SshYumRepoChecker.repoOutputUsesEndpoint( + "${ZSTACK_REPO}:baseurl=not-a-url\n${QEMU_REPO}:baseurl=http://[2001:db8::10]:8080/repo\n", + "2001:db8::10", 8080) + } + + private static String repoOutput(String zstackBaseUrl, String qemuBaseUrl) { + return "${ZSTACK_REPO}:baseurl=${zstackBaseUrl}\n${QEMU_REPO}:baseurl=${qemuBaseUrl}\n" + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy b/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy index 2c75a9929af..01c3ca042a3 100644 --- a/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy @@ -95,6 +95,9 @@ class ManagementNetworkIpv6Case extends SubCase { void test() { testSelectManagementServerIpDualStackPolicy() testSelectManagementServerIpSkipsLoopbackAndLinkLocal() + testStrictSelectionWithoutSameFamily() + testStrictSelectionRejectsUnconfiguredRouteSource() + testStrictSelectionReportsRouteFailure() testSelectApplianceVmManagementNodeIpByCidr() testBuildUrlIpv4() testBuildUrlIpv6() @@ -106,6 +109,8 @@ class ManagementNetworkIpv6Case extends SubCase { testKvmAgentUrlsIpv6() testKvmCallbackPrecheckUsesTargetAwareCallbackUrl() testRestFacadeIpv6Urls() + testRestFacadeSelectsSendCommandUrlByTargetIpVersion() + testRestPortFirewallRulesFollowManagementAddressFamilies() testSshTargetUsesRawIpv6Host() testScpTargetUsesBracketedIpv6Host() testCallbackCheckerUsesIpv6Options() @@ -255,7 +260,7 @@ class ManagementNetworkIpv6Case extends SubCase { RESTFacadeImpl.buildCallbackUrl(host, REST_PORT, "zstack") } ] as RESTFacade - String command = KVMHost.buildManagementNodeCallbackCheckCommand(IPV6_2, restf) + String command = KVMHost.buildManagementNodeCallbackCheckCommandForManagementHost(IPV6, restf) String callbackUrl = RESTFacadeImpl.buildCallbackUrl(IPV6, REST_PORT, "zstack") assert callbackUrl == "http://[${IPV6}]:${REST_PORT}/zstack${RESTConstant.CALLBACK_PATH}" @@ -274,6 +279,35 @@ class ManagementNetworkIpv6Case extends SubCase { "http://[2001:db8::1]:8080/zstack${RESTConstant.COMMAND_CHANNEL_PATH}" } + void testRestPortFirewallRulesFollowManagementAddressFamilies() { + List ipv4Rules = [] + List ipv6Rules = [] + + RESTFacadeImpl.installRestPortFirewallRules(REST_PORT, false, + { ipv4Rules.add(it) }, { ipv6Rules.add(it) }) + assert ipv4Rules == ["-A INPUT -p tcp -m state --state NEW -m tcp --dport ${REST_PORT} -j ACCEPT"] + assert ipv6Rules.isEmpty() + + ipv4Rules.clear() + RESTFacadeImpl.installRestPortFirewallRules(REST_PORT, true, + { ipv4Rules.add(it) }, { ipv6Rules.add(it) }) + assert ipv4Rules.size() == 1 + assert ipv6Rules == ipv4Rules + } + + void testRestFacadeSelectsSendCommandUrlByTargetIpVersion() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + String defaultUrl = "http://${IPV4}:${REST_PORT}/zstack${RESTConstant.COMMAND_CHANNEL_PATH}" + + assert RESTFacadeImpl.selectSendCommandUrl(IPV6_2, defaultUrl, REST_PORT, "zstack") == + "http://[${IPV6}]:${REST_PORT}/zstack${RESTConstant.COMMAND_CHANNEL_PATH}" + assert RESTFacadeImpl.selectSendCommandUrl("host.example.com", defaultUrl, REST_PORT, "zstack") == defaultUrl + } + } + void testRestFacadeSelectsCallbackUrlByTargetIpVersion() { withManagementServerIpProperties([ "management.server.ip" : IPV4, @@ -432,6 +466,62 @@ class ManagementNetworkIpv6Case extends SubCase { } } + void testStrictSelectionWithoutSameFamily() { + def selection + withManagementServerIpProperties([ + "management.server.ip": IPV4, + ]) { + selection = Platform.selectManagementServerIpForRemoteStrict( + IPV6_2, IPV6_2, null) + + assert !selection.success + assert selection.failureReason.toString() == "NO_SAME_FAMILY_MGT" + assert selection.selectedIp == null + assert selection.remoteIp == IPV6_2 + assert selection.managementServerIps == [IPV4] + } + assert Platform.managementServerIpSelectionError(selection).globalErrorCode == + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000" + } + + void testStrictSelectionRejectsUnconfiguredRouteSource() { + String unconfiguredRouteSource = "2001:db8::88" + def selection + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + selection = Platform.selectManagementServerIpForRemoteStrict( + IPV6_2, unconfiguredRouteSource, null) + + assert !selection.success + assert selection.failureReason.toString() == "ROUTE_SOURCE_NOT_MANAGEMENT" + assert selection.routeSourceIp == unconfiguredRouteSource + assert selection.managementServerIps == [IPV4, IPV6] + } + assert Platform.managementServerIpSelectionError(selection).globalErrorCode == + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002" + } + + void testStrictSelectionReportsRouteFailure() { + String routeError = "exitCode=2, stderr=network unreachable" + def selection + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + selection = Platform.selectManagementServerIpForRemoteStrict( + IPV6_2, null, routeError) + + assert !selection.success + assert selection.failureReason.toString() == "ROUTE_LOOKUP_FAILED" + assert selection.routeError == routeError + } + def error = Platform.managementServerIpSelectionError(selection) + assert error.globalErrorCode == "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001" + assert error.formatArgs.any { it.contains(routeError) } + } + void testManagementServerSecondaryPropertyRejectsWrongAddressFamily() { withManagementServerIpProperties([ "management.server.ip" : IPV4, diff --git a/test/src/test/groovy/org/zstack/test/integration/core/ansible/AnsibleRunnerTargetAwareArgumentsCase.groovy b/test/src/test/groovy/org/zstack/test/integration/core/ansible/AnsibleRunnerTargetAwareArgumentsCase.groovy new file mode 100644 index 00000000000..a386de35014 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/core/ansible/AnsibleRunnerTargetAwareArgumentsCase.groovy @@ -0,0 +1,256 @@ +package org.zstack.test.integration.core.ansible + +import org.junit.Test +import org.zstack.core.CoreGlobalProperty +import org.zstack.core.Platform +import org.zstack.core.ansible.AnsibleRunner +import org.zstack.core.ansible.CallBackNetworkChecker +import org.zstack.core.ansible.RunAnsibleMsg +import org.zstack.core.cloudbus.CloudBus +import org.zstack.header.core.ReturnValueCompletion +import org.zstack.header.errorcode.ErrorCode +import org.zstack.header.rest.RESTFacade + +import java.lang.reflect.Field +import java.lang.reflect.InvocationHandler +import java.lang.reflect.Method +import java.lang.reflect.Proxy + +class AnsibleRunnerTargetAwareArgumentsCase { + private static final String IPV4 = "127.0.0.1" + private static final String IPV4_TARGET = "127.0.0.2" + private static final String IPV6 = "::1" + private static final String IPV6_TARGET = "::1" + private static final int REST_PORT = 8080 + + @Test + void test() { + String oldUserHome = System.getProperty("user.home") + File testHome = File.createTempDir("zstack-ansible-runner", "") + try { + System.setProperty("user.home", testHome.absolutePath) + testPrimaryIpv4TargetIpv6() + testPrimaryIpv6TargetIpv4() + testHostnameTargetKeepsRestFacadeHost() + testInvalidTargetKeepsRestFacadeHost() + testCallbackCheckerUsesSameSelectedHost() + testNoSameFamilyReturnsErrorWithoutMessage() + } finally { + System.setProperty("user.home", oldUserHome) + testHome.deleteDir() + } + } + + void testPrimaryIpv4TargetIpv6() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + RunAnsibleMsg msg = runAnsible(IPV6_TARGET, IPV4, "http://${IPV4}:${REST_PORT}") + + assert msg.targetIp == IPV6_TARGET + assert msg.deployArguments.pipUrl == "http://[${IPV6}]:${REST_PORT}/zstack/static/pypi/simple" + assert msg.deployArguments.trustedHost == IPV6 + assert msg.deployArguments.yumServer == "[${IPV6}]:${REST_PORT}" + } + } + + void testPrimaryIpv6TargetIpv4() { + withManagementServerIpProperties([ + "management.server.ip" : IPV6, + "management.server.ip4": IPV4, + ]) { + RunAnsibleMsg msg = runAnsible(IPV4_TARGET, IPV6, "http://[${IPV6}]:${REST_PORT}") + + assert msg.targetIp == IPV4_TARGET + assert msg.deployArguments.pipUrl == "http://${IPV4}:${REST_PORT}/zstack/static/pypi/simple" + assert msg.deployArguments.trustedHost == IPV4 + assert msg.deployArguments.yumServer == "${IPV4}:${REST_PORT}" + } + } + + void testHostnameTargetKeepsRestFacadeHost() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + String restHost = "mn.example.com" + RunAnsibleMsg msg = runAnsible("host.example.com", restHost, "http://${restHost}:${REST_PORT}") + + assert msg.deployArguments.pipUrl == "http://${restHost}:${REST_PORT}/zstack/static/pypi/simple" + assert msg.deployArguments.trustedHost == restHost + assert msg.deployArguments.yumServer == "${restHost}:${REST_PORT}" + } + } + + void testInvalidTargetKeepsRestFacadeHost() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + String restHost = "mn.example.com" + RunAnsibleMsg msg = runAnsible("not-an-ip!!", restHost, + "http://${restHost}:${REST_PORT}") + + assert msg.deployArguments.trustedHost == restHost + } + } + + void testNoSameFamilyReturnsErrorWithoutMessage() { + withManagementServerIpProperties([ + "management.server.ip": IPV4, + ]) { + def selection = Platform.selectManagementServerIpForRemoteStrict( + IPV6_TARGET, IPV6_TARGET, null) + def result = runAnsibleResult(IPV6_TARGET, IPV4, + "http://${IPV4}:${REST_PORT}", selection) + + assert result.message == null + assert result.error != null + assert result.error.globalErrorCode == + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000" + } + } + + void testCallbackCheckerUsesSameSelectedHost() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + def result = runAnsibleResult(IPV6_TARGET, IPV4, + "http://${IPV4}:${REST_PORT}", null) + + assert result.error == null + assert result.callbackChecker.callbackIp == IPV6 + } + } + + private RunAnsibleMsg runAnsible(String targetIp, String restHost, String baseUrl) { + def result = runAnsibleResult(targetIp, restHost, baseUrl, null) + assert result.error == null: result.error + assert result.message != null + return result.message as RunAnsibleMsg + } + + private Map runAnsibleResult(String targetIp, String restHost, String baseUrl, def selection) { + RunAnsibleMsg[] captured = new RunAnsibleMsg[1] + ErrorCode[] capturedError = new ErrorCode[1] + CloudBus bus = proxy(CloudBus.class) { Method method, Object[] args -> + if (method.name == "makeTargetServiceIdByResourceUuid") { + return null + } + if (method.name == "send" && args?.length == 2 && args[0] instanceof RunAnsibleMsg) { + captured[0] = args[0] as RunAnsibleMsg + return null + } + return defaultValue(method.returnType) + } + RESTFacade restf = proxy(RESTFacade.class) { Method method, Object[] args -> + switch (method.name) { + case "getBaseUrl": + return baseUrl + case "getHostName": + return restHost + case "getPort": + return REST_PORT + default: + return defaultValue(method.returnType) + } + } + + boolean oldUnitTestOn = CoreGlobalProperty.UNIT_TEST_ON + try { + CoreGlobalProperty.UNIT_TEST_ON = true + AnsibleRunner runner = new AnsibleRunner() + setField(runner, "bus", bus) + setField(runner, "restf", restf) + runner.setForceRun(true) + runner.setTargetIp(targetIp) + runner.setTargetUuid(Platform.uuid) + runner.setPlayBookPath("/tmp/zstack-test-playbook.yaml") + runner.setUsername("root") + runner.setPassword("password") + CallBackNetworkChecker callbackChecker = new CallBackNetworkChecker() + callbackChecker.setCallbackIp("legacy-selection") + runner.installChecker(callbackChecker) + if (selection != null) { + runner.setManagementServerIpSelection(selection) + } + + ReturnValueCompletion completion = new ReturnValueCompletion(null) { + @Override + void success(Boolean returnValue) { + } + + @Override + void fail(ErrorCode errorCode) { + capturedError[0] = errorCode + } + } + runner.run(completion) + + return [message: captured[0], error: capturedError[0], + callbackChecker: callbackChecker] + } finally { + CoreGlobalProperty.UNIT_TEST_ON = oldUnitTestOn + } + } + + private static T proxy(Class type, Closure handler) { + InvocationHandler invocationHandler = { Object ignored, Method method, Object[] args -> + handler.call(method, args) + } as InvocationHandler + return Proxy.newProxyInstance(type.classLoader, [type] as Class[], invocationHandler) as T + } + + private static Object defaultValue(Class returnType) { + if (!returnType.isPrimitive()) { + return null + } + if (returnType == boolean.class) { + return false + } + if (returnType == char.class) { + return (char) 0 + } + return 0 + } + + private static void setField(Object target, String name, Object value) { + Field field = target.class.getDeclaredField(name) + field.setAccessible(true) + field.set(target, value) + } + + private void withManagementServerIpProperties(Map properties, Closure closure) { + List managedKeys = [ + "management.server.ip", + "management.server.ip4", + "management.server.ip6", + ] + Map oldValues = [:] + managedKeys.each { key -> oldValues[key] = System.getProperty(key) } + + try { + resetCachedManagementServerIp() + managedKeys.each { System.clearProperty(it) } + properties.each { key, value -> System.setProperty(key, value) } + closure.call() + } finally { + managedKeys.each { key -> + if (oldValues[key] == null) { + System.clearProperty(key) + } else { + System.setProperty(key, oldValues[key]) + } + } + resetCachedManagementServerIp() + } + } + + private static void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.setAccessible(true) + field.set(null, null) + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/core/salt/SaltDualStackConfigCase.groovy b/test/src/test/groovy/org/zstack/test/integration/core/salt/SaltDualStackConfigCase.groovy new file mode 100644 index 00000000000..883fab17224 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/core/salt/SaltDualStackConfigCase.groovy @@ -0,0 +1,112 @@ +package org.zstack.test.integration.core.salt + +import org.zstack.core.Platform +import org.zstack.core.salt.SaltFacadeImpl +import org.zstack.core.salt.SaltSetupMinionJob +import org.zstack.header.core.ReturnValueCompletion +import org.zstack.header.errorcode.ErrorCode +import org.zstack.testlib.SubCase +import org.zstack.utils.path.PathUtil + +import java.lang.reflect.Field + +class SaltDualStackConfigCase extends SubCase { + @Override + void setup() { + } + + @Override + void environment() { + } + + @Override + void clean() { + } + + @Override + void test() { + testMasterListensOnDualStackSocket() + testMinionSelectsReachableSaltMasterFamily() + testStrictSelectionFailure() + } + + void testMasterListensOnDualStackSocket() { + String masterConfig = PathUtil.findFileOnClassPath("salt/master", true).text + String dualStack = SaltFacadeImpl.renderMasterConfig(masterConfig, 1024, 10, true) + assert dualStack.readLines().contains("interface: '::'") + assert dualStack.readLines().contains("ipv6: true") + + String ipv4Only = SaltFacadeImpl.renderMasterConfig(masterConfig, 1024, 10, false) + assert ipv4Only.readLines().contains("interface: 0.0.0.0") + assert ipv4Only.readLines().contains("ipv6: false") + } + + void testMinionSelectsReachableSaltMasterFamily() { + withManagementServerIpProperties([ + "management.server.ip" : "192.168.1.10", + "management.server.ip6": "2001:db8::10", + ]) { + String template = "master: {managementNodeIp}\nipv6: {ipv6Enabled}\nid: {minionId}\n" + + assert SaltSetupMinionJob.renderMinionConfig(template, "2001:db8::10", "ipv6-minion") == + "master: 2001:db8::10\nipv6: true\nid: ipv6-minion\n" + assert SaltSetupMinionJob.renderMinionConfig(template, "192.168.1.10", "ipv4-minion") == + "master: 192.168.1.10\nipv6: false\nid: ipv4-minion\n" + } + } + + void testStrictSelectionFailure() { + withManagementServerIpProperties([ + "management.server.ip": "127.0.0.1", + ]) { + def selection = Platform.selectManagementServerIpForRemoteStrict( + "::1", "::1", null) + SaltSetupMinionJob job = new SaltSetupMinionJob() + job.setTargetIp("::1") + job.setManagementServerIpSelection(selection) + ErrorCode[] captured = new ErrorCode[1] + + job.run(new ReturnValueCompletion(null) { + @Override + void success(Object ignored) { + assert false: "selection failure must not succeed" + } + + @Override + void fail(ErrorCode errorCode) { + captured[0] = errorCode + } + }) + + assert captured[0] != null + assert captured[0].globalErrorCode == + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000" + } + } + + private static void withManagementServerIpProperties(Map properties, Closure closure) { + List keys = ["management.server.ip", "management.server.ip4", "management.server.ip6"] + Map oldValues = keys.collectEntries { [(it): System.getProperty(it)] } + try { + keys.each { System.clearProperty(it) } + properties.each { key, value -> System.setProperty(key, value) } + resetCachedManagementServerIp() + closure.call() + } finally { + keys.each { key -> + if (oldValues[key] == null) { + System.clearProperty(key) + } else { + System.setProperty(key, oldValues[key]) + } + } + resetCachedManagementServerIp() + } + } + + private static void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.accessible = true + field.set(null, null) + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/host/DualStackAddKvmHostCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/host/DualStackAddKvmHostCase.groovy new file mode 100644 index 00000000000..60d75c8cb57 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/host/DualStackAddKvmHostCase.groovy @@ -0,0 +1,463 @@ +package org.zstack.test.integration.kvm.host + +import org.zstack.core.Platform +import org.zstack.core.CoreGlobalProperty +import org.zstack.core.cloudbus.CloudBus +import org.zstack.core.db.DatabaseFacade +import org.zstack.core.db.Q +import org.zstack.header.host.HostStatus +import org.zstack.header.host.HostConstant +import org.zstack.header.host.PingHostMsg +import org.zstack.header.host.PingHostReply +import org.zstack.header.host.HostVO +import org.zstack.header.host.HostVO_ +import org.zstack.header.rest.RESTFacade +import org.zstack.header.rest.RESTConstant +import org.zstack.header.tag.SystemTagVO +import org.zstack.header.tag.SystemTagVO_ +import org.zstack.header.zone.ZoneVO +import org.zstack.sdk.AddKVMHostAction +import org.zstack.sdk.ClusterInventory +import org.zstack.header.vm.PauseVmOnHypervisorMsg +import org.zstack.header.vm.VmInstanceInventory +import org.zstack.test.integration.kvm.KvmTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import org.zstack.core.ansible.RunAnsibleMsg +import org.zstack.core.ansible.RunAnsibleReply +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KVMConstant +import org.zstack.utils.ShellUtils +import org.zstack.utils.gson.JSONObjectUtil +import org.springframework.http.HttpEntity + +import java.lang.reflect.Field +import java.util.concurrent.TimeUnit + +class DualStackAddKvmHostCase extends SubCase { + private static final String UNIT_TEST_AGENT_HOST = "KVMHost.unitTestAgentHost" + private static final String TEST_INTERFACE = "z86567dummy" + private static final String IPV4_MN = "10.254.86.1" + private static final String IPV4_HOST = "10.254.86.2" + private static final String IPV4_ONLY_HOST = "10.254.86.3" + private static final String IPV6_MN = "fd86:567::1" + private static final String IPV6_MN_ALT = "fd86:567::10" + private static final String IPV6_HOST = "fd86:567::2" + private static final String IPV6_ONLY_HOST = "fd86:567::3" + private static final String IPV6_FAILURE_HOST = "fd86:567::30" + private static final String IPV6_ROUTE_FAILURE_HOST = "fd86:568::2" + + EnvSpec env + ClusterInventory cluster + List oldChronyServers + + @Override + void setup() { + deleteTestInterface() + runNetworkCommand("ip link add ${TEST_INTERFACE} type dummy") + runNetworkCommand("ip link set ${TEST_INTERFACE} up") + runNetworkCommand("ip -4 addr add ${IPV4_MN}/24 dev ${TEST_INTERFACE}") + runNetworkCommand("ip -6 addr add ${IPV6_MN}/64 dev ${TEST_INTERFACE}") + useSpring(KvmTest.springSpec) + } + + @Override + void environment() { + env = HostEnv.noHostBasicEnv() + } + + @Override + void clean() { + System.clearProperty(UNIT_TEST_AGENT_HOST) + CoreGlobalProperty.CHRONY_SERVERS = oldChronyServers + try { + if (env != null) { + env.delete() + } + } finally { + deleteTestInterface() + } + } + + @Override + void test() { + env.create { + cluster = env.inventoryByName("cluster") as ClusterInventory + oldChronyServers = CoreGlobalProperty.CHRONY_SERVERS + try { + System.setProperty(UNIT_TEST_AGENT_HOST, "127.0.0.1") + CoreGlobalProperty.CHRONY_SERVERS = ["127.0.0.1"] + + testPrimaryIpv4AddIpv6Host() + testPrimaryIpv6AddIpv4Host() + testIpv4OnlyRegression() + testIpv6OnlyRegression() + testHostnameRegression() + testPingReResolvesChangedRouteSource() + testPingRouteLookupFailureDoesNotDisconnectHost() + testAnsibleFailureRollsBackHost() + testNoSameFamilyRollsBackWithoutChecker() + testUnconfiguredRouteSourceRollsBackWithoutChecker() + } finally { + System.clearProperty(UNIT_TEST_AGENT_HOST) + CoreGlobalProperty.CHRONY_SERVERS = oldChronyServers + deleteTestInterface() + } + } + } + + void testPrimaryIpv4AddIpv6Host() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4_MN, + "management.server.ip6": IPV6_MN, + ]) { + updateZoneIpVersion("ipv6") + def captured = addHostAndCaptureCommands(IPV6_HOST, "dual-stack-ipv6-host") + RunAnsibleMsg msg = captured.ansible + + assert msg.targetIp == IPV6_HOST + assert msg.deployArguments.pipUrl.contains("[${IPV6_MN}]") + assert msg.deployArguments.trustedHost == IPV6_MN + assert msg.deployArguments.yumServer.startsWith("[${IPV6_MN}]:") + assert captured.connect.sendCommandUrl == bean(RESTFacade.class).buildSendCommandUrl(IPV6_HOST) + assertPingRepairsTargetAwareSendCommandUrl(msg.targetUuid, IPV6_MN) + deleteHost { uuid = msg.targetUuid } + } + } + + void testPrimaryIpv6AddIpv4Host() { + withManagementServerIpProperties([ + "management.server.ip" : IPV6_MN, + "management.server.ip4": IPV4_MN, + ]) { + updateZoneIpVersion("ipv4") + def captured = addHostAndCaptureCommands(IPV4_HOST, "dual-stack-ipv4-host") + RunAnsibleMsg msg = captured.ansible + + assert msg.targetIp == IPV4_HOST + assert msg.deployArguments.pipUrl.contains("//${IPV4_MN}:") + assert msg.deployArguments.trustedHost == IPV4_MN + assert msg.deployArguments.yumServer.startsWith("${IPV4_MN}:") + assert captured.connect.sendCommandUrl == bean(RESTFacade.class).buildSendCommandUrl(IPV4_HOST) + deleteHost { uuid = msg.targetUuid } + } + } + + void testIpv4OnlyRegression() { + withManagementServerIpProperties(["management.server.ip": IPV4_MN]) { + updateZoneIpVersion("ipv4") + def captured = addHostAndCaptureCommands(IPV4_ONLY_HOST, "ipv4-only-host") + RunAnsibleMsg msg = captured.ansible + + assert msg.deployArguments.trustedHost == IPV4_MN + assert captured.connect.sendCommandUrl == bean(RESTFacade.class).buildSendCommandUrl(IPV4_ONLY_HOST) + deleteHost { uuid = msg.targetUuid } + } + } + + void testIpv6OnlyRegression() { + withManagementServerIpProperties(["management.server.ip": IPV6_MN]) { + updateZoneIpVersion("ipv6") + def captured = addHostAndCaptureCommands(IPV6_ONLY_HOST, "ipv6-only-host") + RunAnsibleMsg msg = captured.ansible + + assert msg.deployArguments.trustedHost == IPV6_MN + assert msg.deployArguments.yumServer.startsWith("[${IPV6_MN}]:") + assert captured.connect.sendCommandUrl == bean(RESTFacade.class).buildSendCommandUrl(IPV6_ONLY_HOST) + deleteHost { uuid = msg.targetUuid } + } + } + + void testHostnameRegression() { + withManagementServerIpProperties(["management.server.ip": IPV4_MN]) { + updateZoneIpVersion("ipv4") + def captured = addHostAndCaptureCommands("localhost", "hostname-host") + RunAnsibleMsg msg = captured.ansible + + assert msg.targetIp == "localhost" + assert msg.deployArguments.trustedHost == bean(RESTFacade.class).hostName + assert captured.connect.sendCommandUrl == bean(RESTFacade.class).buildSendCommandUrl("localhost") + deleteHost { uuid = msg.targetUuid } + } + } + + void testPingReResolvesChangedRouteSource() { + withManagementServerIpProperties([ + "management.server.ip" : IPV6_MN, + "management.server.ip6": IPV6_MN_ALT, + ]) { + updateZoneIpVersion("ipv6") + runNetworkCommand("ip -6 addr add ${IPV6_MN_ALT}/64 dev ${TEST_INTERFACE}") + runNetworkCommand("ip -6 route replace ${IPV6_HOST}/128 dev ${TEST_INTERFACE} src ${IPV6_MN}") + try { + def captured = addHostAndCaptureCommands(IPV6_HOST, "route-switch-host") + RunAnsibleMsg msg = captured.ansible + assert msg.deployArguments.trustedHost == IPV6_MN + + runNetworkCommand("ip -6 route replace ${IPV6_HOST}/128 dev ${TEST_INTERFACE} src ${IPV6_MN_ALT}") + assertPingRepairsTargetAwareSendCommandUrl(msg.targetUuid, IPV6_MN_ALT) + assertKvmAsyncCallbackUsesManagementHost(msg.targetUuid, IPV6_MN_ALT) + deleteHost { uuid = msg.targetUuid } + } finally { + ShellUtils.runAndReturn("ip -6 route delete ${IPV6_HOST}/128 dev ${TEST_INTERFACE}", true) + ShellUtils.runAndReturn("ip -6 addr delete ${IPV6_MN_ALT}/64 dev ${TEST_INTERFACE}", true) + } + } + } + + void testAnsibleFailureRollsBackHost() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4_MN, + "management.server.ip6": IPV6_MN, + ]) { + updateZoneIpVersion("ipv6") + String hostUuid = Platform.uuid + boolean ansibleCalled = false + env.message(RunAnsibleMsg.class) { RunAnsibleMsg msg, bus -> + ansibleCalled = true + bus.replyErrorByMessageType(msg, "simulated ansible failure") + } + + def result = addHost(hostUuid, IPV6_FAILURE_HOST, "ansible-failure-host") + + assert ansibleCalled + assert result.error != null + assert !Q.New(HostVO.class).eq(HostVO_.uuid, hostUuid).isExists() + } + } + + void testPingRouteLookupFailureDoesNotDisconnectHost() { + withManagementServerIpProperties([ + "management.server.ip": IPV6_MN, + ]) { + updateZoneIpVersion("ipv6") + runNetworkCommand("ip -6 route replace ${IPV6_ROUTE_FAILURE_HOST}/128 " + + "dev ${TEST_INTERFACE} src ${IPV6_MN}") + try { + def captured = addHostAndCaptureCommands( + IPV6_ROUTE_FAILURE_HOST, "transient-route-failure-host") + String hostUuid = captured.ansible.targetUuid + env.afterSimulator(KVMConstant.KVM_PING_PATH) { + KVMAgentCommands.PingResponse rsp, HttpEntity entity -> + KVMAgentCommands.PingCmd cmd = JSONObjectUtil.toObject( + entity.body, KVMAgentCommands.PingCmd.class) + if (cmd.hostUuid == hostUuid) { + rsp.hostUuid = cmd.hostUuid + rsp.version = bean(DatabaseFacade.class).dbVersion + rsp.sendCommandUrl = "http://stale.example.com/command" + } + return rsp + } + runNetworkCommand("ip -6 route replace unreachable ${IPV6_ROUTE_FAILURE_HOST}/128") + + PingHostMsg msg = new PingHostMsg() + msg.hostUuid = hostUuid + CloudBus bus = bean(CloudBus.class) + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid) + PingHostReply reply = bus.call(msg) as PingHostReply + + assert reply.success + TimeUnit.MILLISECONDS.sleep(500) + HostVO host = Q.New(HostVO.class).eq(HostVO_.uuid, hostUuid).find() + assert host.status == HostStatus.Connected + deleteHost { uuid = hostUuid } + } finally { + ShellUtils.runAndReturn( + "ip -6 route delete ${IPV6_ROUTE_FAILURE_HOST}/128", true) + } + } + } + + void testNoSameFamilyRollsBackWithoutChecker() { + withManagementServerIpProperties([ + "management.server.ip": "127.0.0.1", + ]) { + updateZoneIpVersion("ipv6") + assertSelectionFailureRollsBackWithoutAnsible( + IPV6_HOST, + "no-same-family-host", + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000") + } + } + + void testUnconfiguredRouteSourceRollsBackWithoutChecker() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4_MN, + "management.server.ip6": "fd86:567::10", + ]) { + updateZoneIpVersion("ipv6") + assertSelectionFailureRollsBackWithoutAnsible( + IPV6_HOST, + "unconfigured-route-source-host", + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002") + } + } + + private void assertSelectionFailureRollsBackWithoutAnsible(String managementIp, + String name, + String globalErrorCode) { + String hostUuid = Platform.uuid + boolean ansibleCalled = false + env.message(RunAnsibleMsg.class) { RunAnsibleMsg msg, bus -> + ansibleCalled = true + bus.reply(msg, new RunAnsibleReply()) + } + + def result = addHost(hostUuid, managementIp, name) + + assert !ansibleCalled + assert result.error != null + assert result.error.globalErrorCode == globalErrorCode + assert !Q.New(HostVO.class).eq(HostVO_.uuid, hostUuid).isExists() + } + + private Map addHostAndCaptureCommands(String managementIp, String name) { + String hostUuid = Platform.uuid + RunAnsibleMsg[] capturedAnsible = new RunAnsibleMsg[1] + KVMAgentCommands.ConnectCmd[] capturedConnect = new KVMAgentCommands.ConnectCmd[1] + env.message(RunAnsibleMsg.class) { RunAnsibleMsg msg, bus -> + capturedAnsible[0] = msg + bus.reply(msg, new RunAnsibleReply()) + } + env.afterSimulator(KVMConstant.KVM_CONNECT_PATH) { rsp, HttpEntity entity -> + KVMAgentCommands.ConnectCmd cmd = JSONObjectUtil.toObject( + entity.body, KVMAgentCommands.ConnectCmd.class) + if (cmd.hostUuid == hostUuid) { + capturedConnect[0] = cmd + } + return rsp + } + + def result = addHost(hostUuid, managementIp, name) + + assert result.error == null: "${result.error?.code} ${result.error?.globalErrorCode} ${result.error?.details}" + assert capturedAnsible[0] != null + assert capturedConnect[0] != null + HostVO host = Q.New(HostVO.class).eq(HostVO_.uuid, hostUuid).find() + assert host != null + assert host.status == HostStatus.Connected + return [ansible: capturedAnsible[0], connect: capturedConnect[0]] + } + + private void assertPingRepairsTargetAwareSendCommandUrl(String hostUuid, String managementHost) { + RESTFacade restf = bean(RESTFacade.class) + DatabaseFacade dbf = bean(DatabaseFacade.class) + KVMAgentCommands.UpdateHostConfigurationCmd[] captured = + new KVMAgentCommands.UpdateHostConfigurationCmd[1] + + env.afterSimulator(KVMConstant.KVM_PING_PATH) { + KVMAgentCommands.PingResponse rsp, HttpEntity entity -> + KVMAgentCommands.PingCmd cmd = JSONObjectUtil.toObject(entity.body, KVMAgentCommands.PingCmd.class) + if (cmd.hostUuid == hostUuid) { + rsp.hostUuid = cmd.hostUuid + rsp.version = dbf.dbVersion + rsp.sendCommandUrl = "http://stale.example.com/command" + } + return rsp + } + env.afterSimulator(KVMConstant.KVM_UPDATE_HOST_CONFIGURATION_PATH) { + KVMAgentCommands.UpdateHostConfigurationResponse rsp, HttpEntity entity -> + KVMAgentCommands.UpdateHostConfigurationCmd cmd = JSONObjectUtil.toObject( + entity.body, KVMAgentCommands.UpdateHostConfigurationCmd.class) + if (cmd.hostUuid == hostUuid) { + captured[0] = cmd + } + return rsp + } + + PingHostMsg msg = new PingHostMsg() + msg.hostUuid = hostUuid + CloudBus bus = bean(CloudBus.class) + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid) + PingHostReply reply = bus.call(msg) as PingHostReply + + assert reply.success + retryInSecs { + assert captured[0] != null + } + assert captured[0].sendCommandUrl == + restf.buildSendCommandUrlForManagementHost(managementHost) + } + + private void assertKvmAsyncCallbackUsesManagementHost(String hostUuid, String managementHost) { + String[] callbackUrl = new String[1] + env.afterSimulator(KVMConstant.KVM_PAUSE_VM_PATH) { rsp, HttpEntity entity -> + callbackUrl[0] = entity.headers.getFirst(RESTConstant.CALLBACK_URL) + return rsp + } + + VmInstanceInventory vm = new VmInstanceInventory() + vm.uuid = Platform.uuid + vm.name = "target-aware-callback-vm" + vm.hostUuid = hostUuid + PauseVmOnHypervisorMsg msg = new PauseVmOnHypervisorMsg() + msg.vmInventory = vm + CloudBus bus = bean(CloudBus.class) + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid) + def reply = bus.call(msg) + + assert reply.success + assert callbackUrl[0] == bean(RESTFacade.class).buildCallbackUrl(managementHost) + } + + private def addHost(String hostUuid, String managementIp, String name) { + AddKVMHostAction action = new AddKVMHostAction() + action.sessionId = adminSession() + action.resourceUuid = hostUuid + action.clusterUuid = cluster.uuid + action.managementIp = managementIp + action.name = name + action.username = "root" + action.password = "password" + return action.call() + } + + private void updateZoneIpVersion(String version) { + SystemTagVO currentTag = Q.New(SystemTagVO.class) + .eq(SystemTagVO_.resourceUuid, cluster.zoneUuid) + .eq(SystemTagVO_.resourceType, ZoneVO.simpleName) + .like(SystemTagVO_.tag, "managementNetwork::ipVersion::%") + .find() + assert currentTag != null + + updateSystemTag { + uuid = currentTag.uuid + tag = "managementNetwork::ipVersion::${version}" + } + } + + private void withManagementServerIpProperties(Map properties, Closure closure) { + List keys = ["management.server.ip", "management.server.ip4", "management.server.ip6"] + Map oldValues = keys.collectEntries { [(it): System.getProperty(it)] } + try { + keys.each { System.clearProperty(it) } + properties.each { key, value -> System.setProperty(key, value) } + resetCachedManagementServerIp() + closure.call() + } finally { + keys.each { key -> + if (oldValues[key] == null) { + System.clearProperty(key) + } else { + System.setProperty(key, oldValues[key]) + } + } + resetCachedManagementServerIp() + } + } + + private static void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.accessible = true + field.set(null, null) + } + + private static void runNetworkCommand(String command) { + def result = ShellUtils.runAndReturn(command, true) + assert result.retCode == 0: "${command}: ${result.stderr}" + } + + private static void deleteTestInterface() { + ShellUtils.runAndReturn("ip link delete ${TEST_INTERFACE}", true) + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/vm/migrate/MigrateVmFromDestitonHostCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/migrate/MigrateVmFromDestitonHostCase.groovy index 5fca7dff6b2..164ea6062ea 100644 --- a/test/src/test/groovy/org/zstack/test/integration/kvm/vm/migrate/MigrateVmFromDestitonHostCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/migrate/MigrateVmFromDestitonHostCase.groovy @@ -2,15 +2,20 @@ package org.zstack.test.integration.kvm.vm.migrate import com.google.gson.Gson import org.springframework.http.HttpEntity +import org.zstack.core.Platform import org.zstack.core.db.Q import org.zstack.core.db.SQL import org.zstack.header.Constants import org.zstack.header.longjob.LongJobState import org.zstack.header.longjob.LongJobVO import org.zstack.header.longjob.LongJobVO_ +import org.zstack.header.host.HostVO +import org.zstack.header.host.HostVO_ import org.zstack.header.vm.APIMigrateVmMsg import org.zstack.header.vm.VmInstanceState import org.zstack.header.vm.VmInstanceVO +import org.zstack.header.rest.RESTConstant +import org.zstack.header.rest.RESTFacade import org.zstack.kvm.KVMAgentCommands import org.zstack.kvm.KVMConstant import org.zstack.kvm.KVMGlobalConfig @@ -24,6 +29,8 @@ import org.zstack.testlib.EnvSpec import org.zstack.testlib.SubCase import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil + +import java.lang.reflect.Field /** * Created by shixin.ruan on 2018/02/10. */ @@ -31,15 +38,21 @@ class MigrateVmFromDestitonHostCase extends SubCase { EnvSpec env VmInstanceInventory vm1 def disconnectHostUuid = [] + Map oldManagementServerIpProperties @Override void clean() { - SQL.New(LongJobVO.class).delete() - env.delete() + try { + SQL.New(LongJobVO.class).delete() + env.delete() + } finally { + restoreManagementServerIpProperties() + } } @Override void setup() { + setManagementServerIpProperties() useSpring(KvmTest.springSpec) spring { ceph() @@ -210,6 +223,11 @@ class MigrateVmFromDestitonHostCase extends SubCase { void testMigrateVmFromDestinationHost() { HostInventory host1 = env.inventoryByName("kvm1") HostInventory host = env.inventoryByName("kvm2") + SQL.New(HostVO.class) + .eq(HostVO_.uuid, host.uuid) + .set(HostVO_.managementIp, "::1") + .update() + host.managementIp = "::1" env.simulator(KVMSecurityGroupBackend.SECURITY_GROUP_CLEANUP_UNUSED_RULE_ON_HOST_PATH) { return new KVMAgentCommands.CleanupUnusedRulesOnHostResponse() @@ -218,8 +236,10 @@ class MigrateVmFromDestitonHostCase extends SubCase { KVMGlobalConfig.MIGRATE_AUTO_CONVERGE.updateValue(true) KVMAgentCommands.MigrateVmCmd cmd = null String huuid = null + String callbackUrl = null env.afterSimulator(KVMConstant.KVM_MIGRATE_VM_PATH) { rsp, HttpEntity entity -> huuid = entity.getHeaders().getFirst(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID) + callbackUrl = entity.headers.getFirst(RESTConstant.CALLBACK_URL) cmd = json(entity.getBody(), KVMAgentCommands.MigrateVmCmd.class) return rsp } @@ -234,6 +254,7 @@ class MigrateVmFromDestitonHostCase extends SubCase { assert cmd.destHostIp == host.managementIp assert cmd.srcHostIp == host1.managementIp assert huuid == host.uuid + assert callbackUrl == bean(RESTFacade.class).buildCallbackUrl(host.managementIp) // confirm migration success VmInstanceVO vo1 @@ -266,6 +287,7 @@ class MigrateVmFromDestitonHostCase extends SubCase { assert !cmd.xbzrle assert cmd.destHostIp == host1.managementIp assert cmd.srcHostIp == host.managementIp + assert callbackUrl == bean(RESTFacade.class).buildCallbackUrl(host.managementIp) retryInSecs { vo1 = dbFindByUuid(vm1.getUuid(), VmInstanceVO.class) @@ -294,6 +316,7 @@ class MigrateVmFromDestitonHostCase extends SubCase { } assert !cmd.autoConverge + assert callbackUrl == bean(RESTFacade.class).buildCallbackUrl(host1.managementIp) } private void migrateUnknownVm(String destHostUuid) { @@ -322,4 +345,33 @@ class MigrateVmFromDestitonHostCase extends SubCase { return Q.New(LongJobVO.class).eq(LongJobVO_.uuid, jobInv.uuid).select(LongJobVO_.state).findValue() == LongJobState.Succeeded } } + + private void setManagementServerIpProperties() { + List keys = ["management.server.ip", "management.server.ip4", "management.server.ip6"] + oldManagementServerIpProperties = keys.collectEntries { [(it): System.getProperty(it)] } + keys.each { System.clearProperty(it) } + System.setProperty("management.server.ip", "127.0.0.1") + System.setProperty("management.server.ip6", "::1") + resetCachedManagementServerIp() + } + + private void restoreManagementServerIpProperties() { + if (oldManagementServerIpProperties == null) { + return + } + oldManagementServerIpProperties.each { key, value -> + if (value == null) { + System.clearProperty(key) + } else { + System.setProperty(key, value) + } + } + resetCachedManagementServerIp() + } + + private static void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.accessible = true + field.set(null, null) + } } diff --git a/utils/src/main/java/org/zstack/utils/IptablesUtils.java b/utils/src/main/java/org/zstack/utils/IptablesUtils.java index fc3e8500b30..94dd7613118 100755 --- a/utils/src/main/java/org/zstack/utils/IptablesUtils.java +++ b/utils/src/main/java/org/zstack/utils/IptablesUtils.java @@ -14,12 +14,20 @@ public static void appendRuleToFilterTable(String rule) { } public static void insertRuleToFilterTable(String rule) { - ShellResult ret = ShellUtils.runAndReturn(String.format("/sbin/iptables-save | grep -- '%s' > /dev/null", rule)); + insertRuleToFilterTable("iptables", rule); + } + + public static void insertRuleToFilterTableV6(String rule) { + insertRuleToFilterTable("ip6tables", rule); + } + + private static void insertRuleToFilterTable(String command, String rule) { + ShellResult ret = ShellUtils.runAndReturn(String.format("/sbin/%s-save | grep -- '%s' > /dev/null", command, rule)); if (ret.getRetCode() == 0) { return; } - ret = ShellUtils.runAndReturn(String.format("/sbin/iptables %s", rule.replace("-A", "-I"))); + ret = ShellUtils.runAndReturn(String.format("/sbin/%s %s", command, rule.replace("-A", "-I"))); ret.raiseExceptionIfFail(); } diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index 9fc175de2c1..cbac287190c 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -1618,6 +1618,12 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_CORE_10001 = "ORG_ZSTACK_CORE_10001"; + public static final String ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000 = "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000"; + + public static final String ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001 = "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001"; + + public static final String ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002 = "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002"; + public static final String ORG_ZSTACK_VMWARE_CASCADE_10000 = "ORG_ZSTACK_VMWARE_CASCADE_10000"; public static final String ORG_ZSTACK_VMWARE_CASCADE_10001 = "ORG_ZSTACK_VMWARE_CASCADE_10001"; @@ -6785,6 +6791,7 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_BAREMETAL_PXESERVER_10036 = "ORG_ZSTACK_BAREMETAL_PXESERVER_10036"; public static final String ORG_ZSTACK_BAREMETAL_PXESERVER_10037 = "ORG_ZSTACK_BAREMETAL_PXESERVER_10037"; + public static final String ORG_ZSTACK_BAREMETAL_PXESERVER_10038 = "ORG_ZSTACK_BAREMETAL_PXESERVER_10038"; public static final String ORG_ZSTACK_TEST_INTEGRATION_V2V_10000 = "ORG_ZSTACK_TEST_INTEGRATION_V2V_10000";