diff --git a/conf/salt/master b/conf/salt/master index 2fa75471373..1433e1ee20c 100755 --- a/conf/salt/master +++ b/conf/salt/master @@ -12,11 +12,11 @@ # # The address of the interface to bind to -#interface: 0.0.0.0 +interface: {masterInterface} # Whether the master should listen for IPv6 connections. If this is set to True, # the interface option must be adjusted too (for example: "interface: '::'") -#ipv6: False +ipv6: {ipv6Enabled} # The tcp port used by the publisher #publish_port: 4505 diff --git a/conf/salt/minion b/conf/salt/minion index 2680178055c..dd4694f5456 100755 --- a/conf/salt/minion +++ b/conf/salt/minion @@ -12,7 +12,7 @@ master: {managementNodeIp} # Set whether the minion should connect to the master via IPv6 -#ipv6: False +ipv6: {ipv6Enabled} # Set the number of seconds to wait before attempting to resolve # the master hostname if name resolution fails. Defaults to 30 seconds. diff --git a/core/src/main/java/org/zstack/core/ManagementServerIpSelection.java b/core/src/main/java/org/zstack/core/ManagementServerIpSelection.java new file mode 100644 index 00000000000..e31bd732e09 --- /dev/null +++ b/core/src/main/java/org/zstack/core/ManagementServerIpSelection.java @@ -0,0 +1,78 @@ +package org.zstack.core; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class ManagementServerIpSelection { + public enum FailureReason { + NO_SAME_FAMILY_MGT, + ROUTE_LOOKUP_FAILED, + ROUTE_SOURCE_NOT_MANAGEMENT + } + + private final String selectedIp; + private final String remoteIp; + private final int remoteFamily; + private final List managementServerIps; + private final String routeSourceIp; + private final String routeError; + private final FailureReason failureReason; + + private ManagementServerIpSelection(String selectedIp, String remoteIp, int remoteFamily, + List managementServerIps, String routeSourceIp, + String routeError, FailureReason failureReason) { + this.selectedIp = selectedIp; + this.remoteIp = remoteIp; + this.remoteFamily = remoteFamily; + this.managementServerIps = Collections.unmodifiableList(new ArrayList<>(managementServerIps)); + this.routeSourceIp = routeSourceIp; + this.routeError = routeError; + this.failureReason = failureReason; + } + + public static ManagementServerIpSelection success(String selectedIp, String remoteIp, int remoteFamily, + List managementServerIps, String routeSourceIp) { + return new ManagementServerIpSelection(selectedIp, remoteIp, remoteFamily, + managementServerIps, routeSourceIp, null, null); + } + + public static ManagementServerIpSelection failure(String remoteIp, int remoteFamily, + List managementServerIps, String routeSourceIp, + String routeError, FailureReason failureReason) { + return new ManagementServerIpSelection(null, remoteIp, remoteFamily, + managementServerIps, routeSourceIp, routeError, failureReason); + } + + public boolean isSuccess() { + return failureReason == null; + } + + public String getSelectedIp() { + return selectedIp; + } + + public String getRemoteIp() { + return remoteIp; + } + + public int getRemoteFamily() { + return remoteFamily; + } + + public List getManagementServerIps() { + return managementServerIps; + } + + public String getRouteSourceIp() { + return routeSourceIp; + } + + public String getRouteError() { + return routeError; + } + + public FailureReason getFailureReason() { + return failureReason; + } +} diff --git a/core/src/main/java/org/zstack/core/Platform.java b/core/src/main/java/org/zstack/core/Platform.java index 34df20e3e12..3363ded2b85 100755 --- a/core/src/main/java/org/zstack/core/Platform.java +++ b/core/src/main/java/org/zstack/core/Platform.java @@ -75,6 +75,9 @@ import static org.zstack.utils.CollectionDSL.e; import static org.zstack.utils.CollectionDSL.map; import static org.zstack.utils.StringDSL.ln; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002; public class Platform { private static final CLogger logger = CLoggerImpl.getLogger(Platform.class); @@ -1391,6 +1394,125 @@ public static String getManagementServerIpForRemote(String remoteIp) { return selectManagementServerIpForRemote(remoteIp, null); } + public static ManagementServerIpSelection resolveManagementServerIpForRemoteStrict(String remoteIp) { + String normalizedRemoteIp = normalizeManagementIp(remoteIp); + int remoteFamily = getIpFamily(normalizedRemoteIp); + List managementServerIps = getManagementServerIps().stream() + .map(Platform::normalizeManagementIp) + .distinct() + .collect(Collectors.toList()); + if (remoteFamily == 0) { + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, null, "remote target is not an IP address", + ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED); + } + boolean hasSameFamily = managementServerIps.stream() + .anyMatch(ip -> getIpFamily(ip) == remoteFamily); + if (!hasSameFamily) { + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, null, null, + ManagementServerIpSelection.FailureReason.NO_SAME_FAMILY_MGT); + } + + String family = remoteFamily == IPv6Constants.IPv6 ? "-6" : "-4"; + Linux.ShellResult ret = Linux.shell(String.format("/sbin/ip %s route get %s", family, normalizedRemoteIp)); + if (ret.getExitCode() != 0) { + String routeError = String.format("exitCode=%s, stdout=%s, stderr=%s", + ret.getExitCode(), ret.getStdout(), ret.getStderr()); + return selectManagementServerIpForRemoteStrict(normalizedRemoteIp, null, routeError); + } + + String routeSourceIp = parseRouteSourceIp(normalizedRemoteIp, ret.getStdout()); + if (routeSourceIp == null) { + return selectManagementServerIpForRemoteStrict(normalizedRemoteIp, null, + String.format("route output has no same-family source: %s", ret.getStdout())); + } + return selectManagementServerIpForRemoteStrict(normalizedRemoteIp, routeSourceIp, null); + } + + public static ManagementServerIpSelection selectManagementServerIpForRemoteStrict( + String remoteIp, String routeSourceIp, String routeError) { + String normalizedRemoteIp = normalizeManagementIp(remoteIp); + String normalizedRouteSourceIp = normalizeManagementIp(routeSourceIp); + int remoteFamily = getIpFamily(normalizedRemoteIp); + List managementServerIps = getManagementServerIps().stream() + .map(Platform::normalizeManagementIp) + .distinct() + .collect(Collectors.toList()); + List sameFamilyIps = managementServerIps.stream() + .filter(ip -> getIpFamily(ip) == remoteFamily) + .collect(Collectors.toList()); + + if (remoteFamily == 0) { + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, normalizedRouteSourceIp, "remote target is not an IP address", + ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED); + } + if (sameFamilyIps.isEmpty()) { + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, normalizedRouteSourceIp, routeError, + ManagementServerIpSelection.FailureReason.NO_SAME_FAMILY_MGT); + } + if (StringUtils.isBlank(normalizedRouteSourceIp)) { + String details = StringUtils.isBlank(routeError) ? "route source is unavailable" : routeError; + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, null, details, + ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED); + } + if (getIpFamily(normalizedRouteSourceIp) != remoteFamily || + !sameFamilyIps.contains(normalizedRouteSourceIp)) { + return ManagementServerIpSelection.failure(normalizedRemoteIp, remoteFamily, + managementServerIps, normalizedRouteSourceIp, routeError, + ManagementServerIpSelection.FailureReason.ROUTE_SOURCE_NOT_MANAGEMENT); + } + return ManagementServerIpSelection.success(normalizedRouteSourceIp, normalizedRemoteIp, + remoteFamily, managementServerIps, normalizedRouteSourceIp); + } + + public static ErrorCode managementServerIpSelectionError(ManagementServerIpSelection selection) { + if (selection == null || selection.isSuccess()) { + throw new IllegalArgumentException("selection must contain a failure"); + } + String context = String.format("remote[%s], family[IPv%s], managementIps%s, routeSource[%s], routeError[%s]", + selection.getRemoteIp(), selection.getRemoteFamily(), selection.getManagementServerIps(), + selection.getRouteSourceIp(), selection.getRouteError()); + switch (selection.getFailureReason()) { + case NO_SAME_FAMILY_MGT: + return operr(ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000, + "cannot find a same-family management server IP for %s", context); + case ROUTE_LOOKUP_FAILED: + return operr(ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001, + "failed to resolve the management server route source for %s", context); + case ROUTE_SOURCE_NOT_MANAGEMENT: + return operr(ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002, + "route source is not a configured management server IP for %s", context); + default: + throw new IllegalArgumentException("selection is successful or has no failure reason"); + } + } + + private static int getIpFamily(String ip) { + if (IPv6NetworkUtils.isIpv6Address(ip)) { + return IPv6Constants.IPv6; + } + if (NetworkUtils.isIpv4Address(ip)) { + return IPv6Constants.IPv4; + } + return 0; + } + + private static String parseRouteSourceIp(String remoteIp, String output) { + String[] tokens = StringUtils.defaultString(output).trim().split("\\s+"); + int remoteFamily = getIpFamily(remoteIp); + for (int i = 0; i < tokens.length - 1; i++) { + if ("src".equals(tokens[i])) { + String sourceIp = normalizeManagementIp(tokens[i + 1]); + return getIpFamily(sourceIp) == remoteFamily ? sourceIp : null; + } + } + return null; + } + public static String selectManagementServerIpForRemote(String remoteIp, String routeSourceIp) { if (StringUtils.isBlank(remoteIp)) { return getManagementServerIp(); diff --git a/core/src/main/java/org/zstack/core/agent/AgentManagerImpl.java b/core/src/main/java/org/zstack/core/agent/AgentManagerImpl.java index 27966a5c73c..1c065158027 100755 --- a/core/src/main/java/org/zstack/core/agent/AgentManagerImpl.java +++ b/core/src/main/java/org/zstack/core/agent/AgentManagerImpl.java @@ -3,6 +3,8 @@ import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.annotation.Autowired; import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.ManagementServerIpSelection; +import org.zstack.core.Platform; import org.zstack.core.ansible.AnsibleNeedRun; import org.zstack.core.ansible.AnsibleRunner; import org.zstack.core.ansible.SshFolderMd5Checker; @@ -52,6 +54,11 @@ public static String buildAgentUrl(String ip, int port, String path) { return String.format(HTTP_URL_FORMAT, IPv6NetworkUtils.formatHostForUrl(ip), port, path); } + public static String buildCommandUrl(RESTFacade restf, ManagementServerIpSelection selection) { + return selection == null ? restf.getSendCommandUrl() : + restf.buildSendCommandUrlForManagementHost(selection.getSelectedIp()); + } + @Autowired private CloudBus bus; @Autowired @@ -114,7 +121,8 @@ public String getName() { }); } - private void connect(final DeployAgentMsg msg, final Completion completion) { + private void connect(final DeployAgentMsg msg, final ManagementServerIpSelection managementServerIpSelection, + final Completion completion) { FlowChain chain = FlowChainBuilder.newShareFlowChain(); chain.setName(String.format("continue-connect-agent-server-%s:%s", msg.getIp(), msg.getAgentPort())); chain.then(new ShareFlow() { @@ -149,7 +157,8 @@ public void fail(ErrorCode errorCode) { @Override public void run(FlowTrigger trigger, Map data) { Map config = new HashMap(); - config.put(AgentConstant.CONFIG_COMMAND_URL, restf.getSendCommandUrl()); + config.put(AgentConstant.CONFIG_COMMAND_URL, + buildCommandUrl(restf, managementServerIpSelection)); if (msg.getConfig() != null) { config.putAll(msg.getConfig()); } @@ -193,6 +202,19 @@ private void deployAgent(final DeployAgentMsg msg, final NoErrorCompletion noErr return; } + final ManagementServerIpSelection managementServerIpSelection; + if (NetworkUtils.isIpAddress(msg.getIp())) { + managementServerIpSelection = Platform.resolveManagementServerIpForRemoteStrict(msg.getIp()); + if (!managementServerIpSelection.isSuccess()) { + reply.setError(Platform.managementServerIpSelectionError(managementServerIpSelection)); + bus.reply(msg, reply); + noErrorCompletion.done(); + return; + } + } else { + managementServerIpSelection = null; + } + try { String agentYamlPath = PathUtil.join(AgentConstant.ANSIBLE_MODULE_PATH, "server", AgentConstant.ANSIBLE_PLAYBOOK_NAME); File agentYaml = PathUtil.findFileOnClassPath(agentYamlPath, true); @@ -254,6 +276,7 @@ public boolean isRunNeed() { runner.setSshPort(msg.getSshPort()); } runner.setTargetIp(msg.getIp()); + runner.setManagementServerIpSelection(managementServerIpSelection); runner.setPlayBookPath(tmpAgentYaml.getAbsolutePath()); runner.run(new ReturnValueCompletion(msg, noErrorCompletion) { @Override @@ -271,7 +294,7 @@ public void run() { } }); - connect(msg, new Completion(msg, noErrorCompletion) { + connect(msg, managementServerIpSelection, new Completion(msg, noErrorCompletion) { @Override public void success() { bus.reply(msg, reply); diff --git a/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java b/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java index bd7e06ec714..ebf33492198 100755 --- a/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java +++ b/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java @@ -4,6 +4,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.Platform; +import org.zstack.core.ManagementServerIpSelection; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; import org.zstack.header.core.Completion; @@ -84,6 +86,7 @@ public class AnsibleRunner { private boolean forceRun; private AnsibleBasicArguments deployArguments; + private ManagementServerIpSelection managementServerIpSelection; public String getAnsibleExecutable() { return ansibleExecutable; @@ -197,6 +200,10 @@ public void setDeployArguments(AnsibleBasicArguments deployArguments) { this.deployArguments = deployArguments; } + public void setManagementServerIpSelection(ManagementServerIpSelection selection) { + this.managementServerIpSelection = selection; + } + public Map getArguments() { return arguments; } @@ -382,6 +389,24 @@ private void cleanup() { public void run(ReturnValueCompletion completion) { try { + String deploymentHost; + if (NetworkUtils.isIpAddress(targetIp)) { + ManagementServerIpSelection selection = managementServerIpSelection == null ? + Platform.resolveManagementServerIpForRemoteStrict(targetIp) : managementServerIpSelection; + if (!selection.isSuccess()) { + completion.fail(Platform.managementServerIpSelectionError(selection)); + return; + } + deploymentHost = selection.getSelectedIp(); + } else { + deploymentHost = restf.getHostName(); + } + for (AnsibleChecker checker : checkers) { + if (checker instanceof ManagementServerIpAwareChecker) { + ((ManagementServerIpAwareChecker) checker).setManagementServerIp(deploymentHost); + } + } + if (!forceRun && !isNeedRun()) { completion.success(false); return; @@ -393,9 +418,9 @@ public void run(ReturnValueCompletion completion) { deployArguments = new AnsibleBasicArguments(); } - deployArguments.setPipUrl(buildPipUrl(restf.getHostName(), port)); - deployArguments.setTrustedHost(restf.getHostName()); - deployArguments.setYumServer(IPv6NetworkUtils.formatHostPort(restf.getHostName(), port)); + deployArguments.setPipUrl(buildPipUrl(deploymentHost, port)); + deployArguments.setTrustedHost(deploymentHost); + deployArguments.setYumServer(IPv6NetworkUtils.formatHostPort(deploymentHost, port)); deployArguments.setRemoteUser(username); if (password != null && !password.isEmpty()) { deployArguments.setRemotePass(password); diff --git a/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java b/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java index 5cf5665c06b..63720a51151 100644 --- a/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java @@ -19,7 +19,7 @@ /** * Created by mingjian.deng on 2019/6/12. */ -public class CallBackNetworkChecker implements AnsibleChecker { +public class CallBackNetworkChecker implements AnsibleChecker, ManagementServerIpAwareChecker { private static final CLogger logger = Utils.getLogger(CallBackNetworkChecker.class); private String username; @@ -136,6 +136,11 @@ public void setCallbackIp(String callbackIp) { this.callbackIp = callbackIp; } + @Override + public void setManagementServerIp(String managementServerIp) { + setCallbackIp(managementServerIp); + } + public int getCallBackPort() { return callBackPort; } diff --git a/core/src/main/java/org/zstack/core/ansible/ManagementServerIpAwareChecker.java b/core/src/main/java/org/zstack/core/ansible/ManagementServerIpAwareChecker.java new file mode 100644 index 00000000000..4d91567f8f9 --- /dev/null +++ b/core/src/main/java/org/zstack/core/ansible/ManagementServerIpAwareChecker.java @@ -0,0 +1,5 @@ +package org.zstack.core.ansible; + +public interface ManagementServerIpAwareChecker { + void setManagementServerIp(String managementServerIp); +} diff --git a/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java b/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java index 8438fce4eac..c446df48a77 100644 --- a/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/SshYumRepoChecker.java @@ -7,22 +7,33 @@ import org.zstack.header.rest.RESTFacade; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; +import org.zstack.utils.network.IPv6NetworkUtils; import org.zstack.utils.ssh.Ssh; import org.zstack.utils.ssh.SshResult; +import java.net.URI; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + /** * Created by GuoYi on 2018-12-24. */ @Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) -public class SshYumRepoChecker implements AnsibleChecker { +public class SshYumRepoChecker implements AnsibleChecker, ManagementServerIpAwareChecker { @Autowired protected RESTFacade restf; private static final CLogger logger = Utils.getLogger(SshYumRepoChecker.class); + private static final String ZSTACK_REPO = "/etc/yum.repos.d/zstack-mn.repo"; + private static final String QEMU_REPO = "/etc/yum.repos.d/qemu-kvm-ev-mn.repo"; + private static final String READ_REPO_BASEURLS = String.format( + "grep -H -E '^[[:space:]]*baseurl[[:space:]]*=' %s %s", ZSTACK_REPO, QEMU_REPO); private String username; private String password; private String privateKey; private String targetIp; + private String managementServerIp; private int sshPort = 22; @Override @@ -36,9 +47,7 @@ public boolean needDeploy() { .setPassword(password).setPort(sshPort) .setHostname(targetIp); try { - ssh.sudoCommand(String.format("sed -i '/baseurl/s/\\([0-9]\\{1,3\\}\\.\\)\\{3\\}[0-9]\\{1,3\\}:\\([0-9]\\+\\)/%s/g' /etc/yum.repos.d/{zstack,qemu-kvm-ev}-mn.repo", - restf.getHostName() + ":" + restf.getPort() - )); + ssh.sudoCommand(repoReadCommand()); SshResult ret = ssh.setTimeout(60).runAndClose(); if (ret.getReturnCode() != 0) { logger.warn(String.format("exec ssh command failed, return code: %d, stdout: %s, stderr: %s", @@ -46,15 +55,71 @@ public boolean needDeploy() { return true; } - ssh.reset(); + if (StringUtils.isBlank(managementServerIp)) { + logger.warn(String.format("management server IP is not set for repo checker on target[%s]", targetIp)); + return true; + } + if (!repoOutputUsesEndpoint(ret.getStdout(), managementServerIp, restf.getPort())) { + logger.debug(String.format("yum repo endpoint on target[%s] does not match management node[%s:%s]", + targetIp, managementServerIp, restf.getPort())); + return true; + } + } catch (Exception e) { + logger.warn(String.format("failed to check yum repo endpoint on target[%s]", targetIp), e); + return true; } finally { ssh.close(); } - logger.debug("successfully configured zstack-mn.repo and qemu-kvm-ev-mn.repo in " + targetIp); + logger.debug("yum repo endpoints match the management node on target " + targetIp); return false; } + static boolean repoOutputUsesEndpoint(String output, String expectedHost, int expectedPort) { + if (StringUtils.isBlank(output) || StringUtils.isBlank(expectedHost)) { + return false; + } + + Set expectedRepos = new HashSet<>(Arrays.asList(ZSTACK_REPO, QEMU_REPO)); + Set matchedRepos = new HashSet<>(); + for (String line : output.split("\\r?\\n")) { + int pathSeparator = line.indexOf(':'); + int valueSeparator = line.indexOf('=', pathSeparator + 1); + if (pathSeparator <= 0 || valueSeparator <= pathSeparator || + !"baseurl".equals(line.substring(pathSeparator + 1, valueSeparator).trim())) { + return false; + } + + String repo = line.substring(0, pathSeparator).trim(); + if (!expectedRepos.contains(repo)) { + return false; + } + + try { + URI baseUrl = new URI(line.substring(valueSeparator + 1).trim()); + String actualHost = IPv6NetworkUtils.stripHostUrlBrackets(baseUrl.getHost()); + String normalizedExpectedHost = IPv6NetworkUtils.isIpv6Address(expectedHost) ? + IPv6NetworkUtils.normalizeIpv6(expectedHost) : expectedHost; + String normalizedActualHost = IPv6NetworkUtils.isIpv6Address(actualHost) ? + IPv6NetworkUtils.normalizeIpv6(actualHost) : actualHost; + if (normalizedActualHost == null || + !normalizedExpectedHost.equalsIgnoreCase(normalizedActualHost) || + baseUrl.getPort() != expectedPort) { + return false; + } + } catch (Exception e) { + return false; + } + matchedRepos.add(repo); + } + + return matchedRepos.containsAll(expectedRepos); + } + + static String repoReadCommand() { + return READ_REPO_BASEURLS; + } + @Override public void deleteDestFile() { @@ -99,4 +164,9 @@ public String getTargetIp() { public void setTargetIp(String targetIp) { this.targetIp = targetIp; } + + @Override + public void setManagementServerIp(String managementServerIp) { + this.managementServerIp = managementServerIp; + } } diff --git a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java index 26dbb7ec675..e3215077f79 100755 --- a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java @@ -62,6 +62,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import static org.zstack.core.Platform.*; import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; @@ -193,7 +194,8 @@ void init() { port = Platform.getManagementNodeServicePort(); - IptablesUtils.insertRuleToFilterTable(String.format("-A INPUT -p tcp -m state --state NEW -m tcp --dport %s -j ACCEPT", port)); + installRestPortFirewallRules(port, Platform.getManagementServerIp6() != null, + IptablesUtils::insertRuleToFilterTable, IptablesUtils::insertRuleToFilterTableV6); if ("AUTO".equals(hostname)) { callbackHostName = Platform.getManagementServerIp(); @@ -215,6 +217,16 @@ void init() { CoreGlobalProperty.REST_FACADE_MAX_TOTAL); } + public static void installRestPortFirewallRules(int port, boolean ipv6Enabled, + Consumer ipv4RuleInstaller, + Consumer ipv6RuleInstaller) { + String rule = String.format("-A INPUT -p tcp -m state --state NEW -m tcp --dport %s -j ACCEPT", port); + ipv4RuleInstaller.accept(rule); + if (ipv6Enabled) { + ipv6RuleInstaller.accept(rule); + } + } + public static String buildBaseUrl(String hostName, int port, String path) { String url = IPv6NetworkUtils.buildHttpUrl(hostName, port); if (path != null && !path.isEmpty()) { @@ -260,6 +272,13 @@ public static String buildSendCommandUrl(String hostName, int port, String path) return ub.build().toUriString(); } + public static String selectSendCommandUrl(String remoteHost, String defaultSendCommandUrl, int port, String path) { + if (!NetworkUtils.isIpAddress(remoteHost)) { + return defaultSendCommandUrl; + } + return buildSendCommandUrl(Platform.getManagementServerIpForRemote(remoteHost), port, path); + } + // cap the agent-advertised keep-alive to keepAliveMs; also cap when the agent sends none (duration < 0) static long cappedKeepAlive(long serverDuration, long capMs) { return (serverDuration < 0 || serverDuration > capMs) ? capMs : serverDuration; @@ -1061,6 +1080,16 @@ public String getSendCommandUrl() { return sendCommandUrl; } + @Override + public String buildSendCommandUrl(String remoteHost) { + return selectSendCommandUrl(remoteHost, sendCommandUrl, port, path); + } + + @Override + public String buildSendCommandUrlForManagementHost(String managementHost) { + return buildSendCommandUrl(managementHost, port, path); + } + @Override public String getCallbackUrl() { return callbackUrl; diff --git a/core/src/main/java/org/zstack/core/salt/SaltFacadeImpl.java b/core/src/main/java/org/zstack/core/salt/SaltFacadeImpl.java index 7f522167ca9..d1a21e1d9ae 100755 --- a/core/src/main/java/org/zstack/core/salt/SaltFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/salt/SaltFacadeImpl.java @@ -6,6 +6,7 @@ import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.Platform; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.MessageSafe; import org.zstack.header.AbstractService; @@ -64,19 +65,25 @@ private boolean deployFile(File src, File dst) throws IOException { private File rewriteMasterConfFile() throws IOException { File masterConfTmpt = PathUtil.findFileOnClassPath(PathUtil.join("salt", SaltConstant.MASTER_CONF_NAME), true); - - Map map = new HashMap(); - map.put("maxOpenFiles", String.valueOf(masterMaxOpenFiles)); - map.put("workerThreads", String.valueOf(masterWorkerThreads)); - String srcConf = FileUtils.readFileToString(masterConfTmpt); - String conf = StringTemplate.substitute(srcConf, map); + boolean ipv6Enabled = Platform.getManagementServerIp6() != null; + String conf = renderMasterConfig(srcConf, masterMaxOpenFiles, masterWorkerThreads, ipv6Enabled); File masterConf = File.createTempFile("zstack-salt", "master"); FileUtils.write(masterConf, conf); return masterConf; } + public static String renderMasterConfig(String template, int maxOpenFiles, int workerThreads, + boolean ipv6Enabled) { + Map map = new HashMap(); + map.put("maxOpenFiles", String.valueOf(maxOpenFiles)); + map.put("workerThreads", String.valueOf(workerThreads)); + map.put("masterInterface", ipv6Enabled ? "'::'" : "0.0.0.0"); + map.put("ipv6Enabled", String.valueOf(ipv6Enabled)); + return StringTemplate.substitute(template, map); + } + private void prepareSaltMaster() throws IOException { @@ -84,6 +91,10 @@ private void prepareSaltMaster() throws IOException { try { IptablesUtils.insertRuleToFilterTable("-A INPUT -p tcp -m state --state NEW -m tcp --dport 4505 -j ACCEPT"); IptablesUtils.insertRuleToFilterTable("-A INPUT -p tcp -m state --state NEW -m tcp --dport 4506 -j ACCEPT"); + if (Platform.getManagementServerIp6() != null) { + IptablesUtils.insertRuleToFilterTableV6("-A INPUT -p tcp -m state --state NEW -m tcp --dport 4505 -j ACCEPT"); + IptablesUtils.insertRuleToFilterTableV6("-A INPUT -p tcp -m state --state NEW -m tcp --dport 4506 -j ACCEPT"); + } File dstMasterConf = new File(PathUtil.join(SaltConstant.SALT_CONF_HOME, SaltConstant.MASTER_CONF_NAME)); if (deployFile(srcMasterConf, dstMasterConf)) { @@ -322,4 +333,3 @@ public void setMasterFileRoots(String masterFileRoots) { this.masterFileRoots = masterFileRoots; } } - diff --git a/core/src/main/java/org/zstack/core/salt/SaltSetupMinionJob.java b/core/src/main/java/org/zstack/core/salt/SaltSetupMinionJob.java index f2b40a50ad2..efeeaace25c 100755 --- a/core/src/main/java/org/zstack/core/salt/SaltSetupMinionJob.java +++ b/core/src/main/java/org/zstack/core/salt/SaltSetupMinionJob.java @@ -6,6 +6,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.Platform; +import org.zstack.core.ManagementServerIpSelection; import org.zstack.core.errorcode.ErrorFacade; import org.zstack.core.job.Job; import org.zstack.core.job.JobContext; @@ -13,6 +14,8 @@ import org.zstack.utils.Utils; import org.zstack.utils.data.StringTemplate; import org.zstack.utils.logging.CLogger; +import org.zstack.utils.network.IPv6NetworkUtils; +import org.zstack.utils.network.NetworkUtils; import org.zstack.utils.path.PathUtil; import org.zstack.utils.ssh.Ssh; import org.zstack.utils.ssh.SshException; @@ -52,32 +55,48 @@ public class SaltSetupMinionJob implements Job { private String minionId; @JobContext private boolean cleanMasterKey; + private ManagementServerIpSelection managementServerIpSelection; @Autowired private ErrorFacade errf; private static final String SALT_BOOTSTRAP = "salt/salt-bootstrap.sh"; - private File rewriteMinionConfFile(String minionId) throws IOException { + private File rewriteMinionConfFile(String minionId, String managementNodeIp) throws IOException { File minionConfTmpt = new File(saltMinionConfPath); - - Map map = new HashMap(); - map.put("managementNodeIp", Platform.getManagementServerIp()); - map.put("minionId", minionId); - String srcConf = FileUtils.readFileToString(minionConfTmpt); - String conf = StringTemplate.substitute(srcConf, map); + String conf = renderMinionConfig(srcConf, managementNodeIp, minionId); File minionConf = File.createTempFile("zstack-salt", "minion"); FileUtils.write(minionConf, conf); return minionConf; } + public static String renderMinionConfig(String template, String managementNodeIp, String minionId) { + Map map = new HashMap(); + map.put("managementNodeIp", managementNodeIp); + map.put("ipv6Enabled", String.valueOf(IPv6NetworkUtils.isIpv6Address(managementNodeIp))); + map.put("minionId", minionId); + return StringTemplate.substitute(template, map); + } + @Override public void run(ReturnValueCompletion completion) { File tmpt = null; Ssh ssh = null; FileInputStream fis = null; try { + String managementNodeIp; + if (NetworkUtils.isIpAddress(targetIp)) { + ManagementServerIpSelection selection = managementServerIpSelection == null ? + Platform.resolveManagementServerIpForRemoteStrict(targetIp) : managementServerIpSelection; + if (!selection.isSuccess()) { + completion.fail(Platform.managementServerIpSelectionError(selection)); + return; + } + managementNodeIp = selection.getSelectedIp(); + } else { + managementNodeIp = Platform.getManagementServerIp(); + } ssh = new Ssh().setHostname(targetIp).setPassword(password).setPrivateKey(privateKey) .setUsername(username).setPort(port); SshResult ret = ssh.checkTool("scp").run(); @@ -100,7 +119,7 @@ public void run(ReturnValueCompletion completion) { logger.debug(String.format("salt-minion is found on system[%s], no need to install new one", targetIp)); } - tmpt = rewriteMinionConfFile(minionId); + tmpt = rewriteMinionConfFile(minionId, managementNodeIp); String minionConfPath = PathUtil.join(SaltConstant.SALT_CONF_HOME, SaltConstant.MINION_CONF_NAME); boolean deployMinion = false; ret = ssh.reset().command(String.format("md5sum %s", minionConfPath)).run(); @@ -167,6 +186,10 @@ public void setTargetIp(String targetIp) { this.targetIp = targetIp; } + public void setManagementServerIpSelection(ManagementServerIpSelection selection) { + this.managementServerIpSelection = selection; + } + public String getUsername() { return username; } diff --git a/header/src/main/java/org/zstack/header/rest/RESTFacade.java b/header/src/main/java/org/zstack/header/rest/RESTFacade.java index 23fee5208c2..0dd4786fc7c 100755 --- a/header/src/main/java/org/zstack/header/rest/RESTFacade.java +++ b/header/src/main/java/org/zstack/header/rest/RESTFacade.java @@ -89,6 +89,10 @@ public interface RESTFacade { String getSendCommandUrl(); + String buildSendCommandUrl(String remoteHost); + + String buildSendCommandUrlForManagementHost(String managementHost); + String getCallbackUrl(); /** diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 226ebbfbcaa..ff84463eb14 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -20,6 +20,7 @@ import org.zstack.header.vm.devices.VmInstanceDeviceManager; import org.zstack.core.CoreGlobalProperty; import org.zstack.core.MessageCommandRecorder; +import org.zstack.core.ManagementServerIpSelection; import org.zstack.core.Platform; import org.zstack.core.agent.AgentConstant; import org.zstack.core.ansible.*; @@ -74,6 +75,7 @@ import org.zstack.header.network.l3.L3NetworkInventory; import org.zstack.header.network.l3.L3NetworkVO; import org.zstack.header.rest.JsonAsyncRESTCallback; +import org.zstack.header.rest.RESTConstant; import org.zstack.header.rest.RESTFacade; import org.zstack.header.storage.primary.*; import org.zstack.header.storage.snapshot.*; @@ -133,6 +135,7 @@ import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; public class KVMHost extends HostBase implements Host { + static final String UNIT_TEST_AGENT_HOST = "KVMHost.unitTestAgentHost"; private static final CLogger logger = Utils.getLogger(KVMHost.class); private static final ZTester tester = Utils.getTester(); private static final String EXTRA_IP_SEPARATOR = ","; @@ -216,6 +219,7 @@ public static Set parseSanIps(String sanOutput) { protected KVMHostFactory factory; @Autowired private RESTFacade restf; + private volatile ManagementServerIpSelection managementServerIpSelection; @Autowired private KVMExtensionEmitter extEmitter; @Autowired @@ -318,7 +322,8 @@ public KVMHost(KVMHostVO self, KVMHostContext context) { super(self); this.context = context; - baseUrl = context.getBaseUrl(); + baseUrl = isUnitTestAgentRedirectEnabled() ? + buildAgentUrl(System.getProperty(UNIT_TEST_AGENT_HOST), "") : context.getBaseUrl(); UriComponentsBuilder ub = UriComponentsBuilder.fromHttpUrl(baseUrl); ub.path(KVMConstant.KVM_CONNECT_PATH); @@ -559,6 +564,7 @@ class Http { Class responseClass; String commandStr; String commandName; + String targetManagementIp; long timeout = -1; Http timeout(long timeout) { @@ -566,6 +572,11 @@ Http timeout(long timeout) { return this; } + Http targetManagementIp(String targetManagementIp) { + this.targetManagementIp = targetManagementIp; + return this; + } + public Http(String path, String cmd, String commandName, Class rspClz) { this.path = path; this.commandStr = cmd; @@ -593,6 +604,17 @@ void call(String resourceUuid, ReturnValueCompletion completion) { Map header = new HashMap<>(); header.put(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID, resourceUuid == null ? self.getUuid() : resourceUuid); + String callbackTarget = targetManagementIp == null ? self.getManagementIp() : targetManagementIp; + if (NetworkUtils.isIpAddress(callbackTarget)) { + ManagementServerIpSelection selection = + Platform.resolveManagementServerIpForRemoteStrict(callbackTarget); + if (!selection.isSuccess()) { + completion.fail(Platform.managementServerIpSelectionError(selection)); + return; + } + header.put(RESTConstant.CALLBACK_URL, + restf.buildCallbackUrl(selection.getSelectedIp())); + } runBeforeAsyncJsonPostExts(header); if (commandStr != null) { long overall = timeoutManager.getTimeout(); @@ -2903,9 +2925,9 @@ public static String buildManagementNodeCallbackCheckCommand(String callbackUrl) return String.format(MANAGEMENT_NODE_CALLBACK_CHECK_COMMAND, callbackUrl, callbackUrl); } - public static String buildManagementNodeCallbackCheckCommand(String hostManagementIp, RESTFacade restf) { - String callbackHost = Platform.getManagementServerIpForRemote(hostManagementIp); - return buildManagementNodeCallbackCheckCommand(restf.buildCallbackUrl(callbackHost)); + public static String buildManagementNodeCallbackCheckCommandForManagementHost( + String managementHost, RESTFacade restf) { + return buildManagementNodeCallbackCheckCommand(restf.buildCallbackUrl(managementHost)); } private static String joinAgentPath(String rootPath, String path) { @@ -2927,7 +2949,14 @@ private static String joinAgentPath(String rootPath, String path) { } private String buildUrl(String path) { - return buildAgentUrl(self.getManagementIp(), path); + String host = isUnitTestAgentRedirectEnabled() ? + System.getProperty(UNIT_TEST_AGENT_HOST) : self.getManagementIp(); + return buildAgentUrl(host, path); + } + + private static boolean isUnitTestAgentRedirectEnabled() { + return CoreGlobalProperty.UNIT_TEST_ON && + StringUtils.isNotBlank(System.getProperty(UNIT_TEST_AGENT_HOST)); } private void executeAsyncHttpCall(final KVMHostAsyncHttpCallMsg msg, final NoErrorCompletion completion) { @@ -3370,7 +3399,8 @@ public void run(final FlowTrigger trigger, Map data) { cmd.vmUuid = vmUuid; String url = buildAgentUrl(dstHostMnIp, KVMConstant.CLEAN_FIRMWARE_FLASH); - new Http<>(url, cmd, AgentResponse.class).call(dstHostUuid, new ReturnValueCompletion(trigger) { + new Http<>(url, cmd, AgentResponse.class).targetManagementIp(dstHostMnIp) + .call(dstHostUuid, new ReturnValueCompletion(trigger) { @Override public void success(AgentResponse ret) { if (!ret.isSuccess()) { @@ -3442,7 +3472,9 @@ protected void scripts() { String migrateUrl = buildAgentUrl( migrateFromDestination ? dstHostMnIp : srcHostMnIp, KVMConstant.KVM_MIGRATE_VM_PATH); - new Http<>(migrateUrl, cmd, MigrateVmResponse.class).call(migrateFromDestination ? dstHostUuid : srcHostUuid, new ReturnValueCompletion(trigger) { + new Http<>(migrateUrl, cmd, MigrateVmResponse.class) + .targetManagementIp(migrateFromDestination ? dstHostMnIp : srcHostMnIp) + .call(migrateFromDestination ? dstHostUuid : srcHostUuid, new ReturnValueCompletion(trigger) { @Override public void success(MigrateVmResponse ret) { if (!ret.isSuccess()) { @@ -3481,7 +3513,8 @@ public void run(final FlowTrigger trigger, Map data) { cmd.hostManagementIp = dstHostMnIp; String url = buildAgentUrl(dstHostMnIp, KVMConstant.KVM_HARDEN_CONSOLE_PATH); - new Http<>(url, cmd, AgentResponse.class).call(dstHostUuid, new ReturnValueCompletion(trigger) { + new Http<>(url, cmd, AgentResponse.class).targetManagementIp(dstHostMnIp) + .call(dstHostUuid, new ReturnValueCompletion(trigger) { @Override public void success(AgentResponse ret) { if (!ret.isSuccess()) { @@ -3514,7 +3547,8 @@ public void run(final FlowTrigger trigger, Map data) { cmd.hostManagementIp = srcHostMnIp; String url = buildAgentUrl(srcHostMnIp, KVMConstant.KVM_DELETE_CONSOLE_FIREWALL_PATH); - new Http<>(url, cmd, AgentResponse.class).call(new ReturnValueCompletion(trigger) { + new Http<>(url, cmd, AgentResponse.class).targetManagementIp(srcHostMnIp) + .call(new ReturnValueCompletion(trigger) { @Override public void success(AgentResponse ret) { if (!ret.isSuccess()) { @@ -5177,7 +5211,7 @@ private void doReconnectHostDueToPingResult(PingResponse ret) { bus.send(rmsg); } - private void doUpdateHostConfiguration() { + private void doUpdateHostConfiguration(ManagementServerIpSelection pingSelection) { thdf.chainSubmit(new ChainTask(null) { @Override public String getSyncSignature() { @@ -5186,9 +5220,29 @@ public String getSyncSignature() { @Override public void run(SyncTaskChain chain) { + ManagementServerIpSelection selection = pingSelection; + if (NetworkUtils.isIpAddress(self.getManagementIp())) { + if (selection == null) { + selection = Platform.resolveManagementServerIpForRemoteStrict(self.getManagementIp()); + } + if (!selection.isSuccess()) { + ErrorCode error = Platform.managementServerIpSelectionError(selection); + logger.warn(String.format("failed to select management server IP for host[uuid:%s]: %s", + self.getUuid(), error)); + changeConnectionState(HostStatusEvent.disconnected); + new HostDisconnectedCanonicalEvent(self.getUuid(), error).fire(); + ReconnectHostMsg rmsg = new ReconnectHostMsg(); + rmsg.setHostUuid(self.getUuid()); + bus.makeTargetServiceIdByResourceUuid(rmsg, HostConstant.SERVICE_ID, self.getUuid()); + bus.send(rmsg); + chain.next(); + return; + } + } UpdateHostConfigurationCmd cmd = new UpdateHostConfigurationCmd(); cmd.hostUuid = self.getUuid(); - cmd.sendCommandUrl = restf.getSendCommandUrl(); + cmd.sendCommandUrl = selection == null ? restf.buildSendCommandUrl(self.getManagementIp()) : + restf.buildSendCommandUrlForManagementHost(selection.getSelectedIp()); restf.asyncJsonPost(updateHostConfigurationPath, cmd, new JsonAsyncRESTCallback(chain) { @Override public void fail(ErrorCode err) { @@ -5267,9 +5321,26 @@ protected void connectHostFailHook(ErrorCode errorCode) { super.connectHostFailHook(errorCode); } - boolean needUpdateHostConfiguration(PingResponse rsp) { + boolean needUpdateHostConfiguration(PingResponse rsp, ManagementServerIpSelection selection) { // host uuid or send command url or version changed - return !restf.getSendCommandUrl().equals(rsp.getSendCommandUrl()); + if (NetworkUtils.isIpAddress(self.getManagementIp())) { + if (selection == null) { + selection = Platform.resolveManagementServerIpForRemoteStrict(self.getManagementIp()); + } + if (!selection.isSuccess()) { + if (selection.getFailureReason() == + ManagementServerIpSelection.FailureReason.ROUTE_LOOKUP_FAILED) { + logger.warn(String.format("skip updating host[uuid:%s] configuration because route lookup " + + "for management IP[%s] failed: %s", self.getUuid(), self.getManagementIp(), + selection.getRouteError())); + return false; + } + return true; + } + } + String sendCommandUrl = selection == null ? restf.buildSendCommandUrl(self.getManagementIp()) : + restf.buildSendCommandUrlForManagementHost(selection.getSelectedIp()); + return !sendCommandUrl.equals(rsp.getSendCommandUrl()); } @Override @@ -5316,8 +5387,14 @@ public void success(PingResponse ret) { } afterDone.add(() -> doReconnectHostDueToPingResult(ret)); - } else if (needUpdateHostConfiguration(ret)) { - afterDone.add(KVMHost.this::doUpdateHostConfiguration); + } else { + ManagementServerIpSelection pingSelection = + NetworkUtils.isIpAddress(self.getManagementIp()) ? + Platform.resolveManagementServerIpForRemoteStrict( + self.getManagementIp()) : null; + if (needUpdateHostConfiguration(ret, pingSelection)) { + afterDone.add(() -> doUpdateHostConfiguration(pingSelection)); + } } trigger.next(); @@ -5448,9 +5525,18 @@ protected int getVmMigrateQuantity() { private ErrorCode connectToAgent() { ErrorCode errCode = null; try { + ManagementServerIpSelection selection = null; + if (NetworkUtils.isIpAddress(self.getManagementIp())) { + selection = Platform.resolveManagementServerIpForRemoteStrict(self.getManagementIp()); + if (!selection.isSuccess()) { + return Platform.managementServerIpSelectionError(selection); + } + managementServerIpSelection = selection; + } ConnectCmd cmd = new ConnectCmd(); cmd.setHostUuid(self.getUuid()); - cmd.setSendCommandUrl(restf.getSendCommandUrl()); + cmd.setSendCommandUrl(selection == null ? restf.buildSendCommandUrl(self.getManagementIp()) : + restf.buildSendCommandUrlForManagementHost(selection.getSelectedIp())); cmd.setIptablesRules(KVMGlobalProperty.IPTABLES_RULES); cmd.setIgnoreMsrs(KVMGlobalConfig.KVM_IGNORE_MSRS.value(Boolean.class)); cmd.setTcpServerPort(KVMGlobalProperty.TCP_SERVER_PORT); @@ -5808,8 +5894,18 @@ public void run(FlowTrigger trigger, Map data) { sshShell.setPassword(getSelf().getPassword()); sshShell.setPort(getSelf().getPort()); sshShell.setWithSudo(false); - final String callbackHost = Platform.getManagementServerIpForRemote(getSelf().getManagementIp()); - final String cmd = buildManagementNodeCallbackCheckCommand(getSelf().getManagementIp(), restf); + String callbackHost = restf.getHostName(); + if (NetworkUtils.isIpAddress(getSelf().getManagementIp())) { + ManagementServerIpSelection selection = + Platform.resolveManagementServerIpForRemoteStrict(getSelf().getManagementIp()); + if (!selection.isSuccess()) { + trigger.fail(Platform.managementServerIpSelectionError(selection)); + return; + } + managementServerIpSelection = selection; + callbackHost = selection.getSelectedIp(); + } + final String cmd = buildManagementNodeCallbackCheckCommandForManagementHost(callbackHost, restf); SshResult ret = sshShell.runCommand(cmd); if (ret.getStderr() != null && ret.getStderr().contains("No route to host")) { // c.f. https://access.redhat.com/solutions/1120533 @@ -6058,12 +6154,25 @@ public void run(FlowTrigger trigger, Map data) { @Override public boolean skip(Map data) { - return CoreGlobalProperty.UNIT_TEST_ON; + return CoreGlobalProperty.UNIT_TEST_ON && !isUnitTestAgentRedirectEnabled(); } @Override public void run(final FlowTrigger trigger, Map data) { - String srcPath = PathUtil.findFileOnClassPath(String.format("ansible/kvm/%s", agentPackageName), true).getAbsolutePath(); + ManagementServerIpSelection selection = null; + String selectedManagementServerIp = restf.getHostName(); + if (NetworkUtils.isIpAddress(getSelf().getManagementIp())) { + selection = Platform.resolveManagementServerIpForRemoteStrict( + getSelf().getManagementIp()); + if (!selection.isSuccess()) { + trigger.fail(Platform.managementServerIpSelectionError(selection)); + return; + } + managementServerIpSelection = selection; + selectedManagementServerIp = selection.getSelectedIp(); + } + String srcPath = isUnitTestAgentRedirectEnabled() ? agentPackageName : + PathUtil.findFileOnClassPath(String.format("ansible/kvm/%s", agentPackageName), true).getAbsolutePath(); String destPath = String.format("/var/lib/zstack/kvm/package/%s", agentPackageName); SshFileMd5Checker checker = new SshFileMd5Checker(); checker.setUsername(getSelf().getUsername()); @@ -6097,7 +6206,7 @@ public void run(final FlowTrigger trigger, Map data) { callbackChecker.setUsername(getSelf().getUsername()); callbackChecker.setPassword(getSelf().getPassword()); callbackChecker.setPort(getSelf().getPort()); - callbackChecker.setCallbackIp(Platform.getManagementServerIpForRemote(getSelf().getManagementIp())); + callbackChecker.setCallbackIp(selectedManagementServerIp); callbackChecker.setCallBackPort(CloudBusGlobalProperty.HTTP_PORT); KvmHostConfigChecker kvmHostConfigChecker = new KvmHostConfigChecker(); @@ -6121,7 +6230,7 @@ public void run(final FlowTrigger trigger, Map data) { hostTcpConnectionCallbackChecker.setUsername(getSelf().getUsername()); hostTcpConnectionCallbackChecker.setPassword(getSelf().getPassword()); hostTcpConnectionCallbackChecker.setPort(getSelf().getPort()); - hostTcpConnectionCallbackChecker.setCallbackIp(Platform.getManagementServerIpForRemote(getSelf().getManagementIp())); + hostTcpConnectionCallbackChecker.setCallbackIp(selectedManagementServerIp); hostTcpConnectionCallbackChecker.setCallBackPort(KVMGlobalProperty.TCP_SERVER_PORT); runner.installChecker(hostTcpConnectionCallbackChecker); } @@ -6133,6 +6242,7 @@ public void run(final FlowTrigger trigger, Map data) { } } runner.setAgentPort(KVMGlobalProperty.AGENT_PORT); + runner.setManagementServerIpSelection(selection); runner.setTargetIp(getSelf().getManagementIp()); runner.setTargetUuid(getSelf().getUuid()); runner.setPlayBookName(KVMConstant.ANSIBLE_PLAYBOOK_NAME); diff --git a/test/src/test/groovy/org/zstack/core/agent/AgentManagerCommandUrlCase.groovy b/test/src/test/groovy/org/zstack/core/agent/AgentManagerCommandUrlCase.groovy new file mode 100644 index 00000000000..ba3ea2c009f --- /dev/null +++ b/test/src/test/groovy/org/zstack/core/agent/AgentManagerCommandUrlCase.groovy @@ -0,0 +1,30 @@ +package org.zstack.core.agent + +import org.junit.Test +import org.zstack.core.ManagementServerIpSelection +import org.zstack.core.rest.RESTFacadeImpl +import org.zstack.header.rest.RESTFacade + +class AgentManagerCommandUrlCase { + private static final String IPV4 = "192.168.1.10" + private static final String IPV6 = "2001:db8::1" + private static final String REMOTE_IPV6 = "2001:db8::2" + private static final int REST_PORT = 8080 + + @Test + void testStrictSelectionAndHostnameFallback() { + RESTFacade restf = [ + getSendCommandUrl: { "http://${IPV4}:${REST_PORT}/zstack/command".toString() }, + buildSendCommandUrlForManagementHost: { String host -> + RESTFacadeImpl.buildSendCommandUrl(host, REST_PORT, "/zstack") + } + ] as RESTFacade + ManagementServerIpSelection selection = ManagementServerIpSelection.success( + IPV6, REMOTE_IPV6, 6, [IPV4, IPV6], IPV6) + + assert AgentManagerImpl.buildCommandUrl(restf, selection) == + "http://[${IPV6}]:${REST_PORT}/zstack/asyncrest/sendcommand" + assert AgentManagerImpl.buildCommandUrl(restf, null) == + "http://${IPV4}:${REST_PORT}/zstack/command" + } +} diff --git a/test/src/test/groovy/org/zstack/core/ansible/SshYumRepoCheckerCase.groovy b/test/src/test/groovy/org/zstack/core/ansible/SshYumRepoCheckerCase.groovy new file mode 100644 index 00000000000..ee466df2b2f --- /dev/null +++ b/test/src/test/groovy/org/zstack/core/ansible/SshYumRepoCheckerCase.groovy @@ -0,0 +1,49 @@ +package org.zstack.core.ansible + +import org.junit.Test + +class SshYumRepoCheckerCase { + private static final String ZSTACK_REPO = "/etc/yum.repos.d/zstack-mn.repo" + private static final String QEMU_REPO = "/etc/yum.repos.d/qemu-kvm-ev-mn.repo" + + @Test + void testRepoEndpoints() { + assert SshYumRepoChecker.repoReadCommand().startsWith("grep ") + assert !SshYumRepoChecker.repoReadCommand().contains("sed -i") + + assert SshYumRepoChecker.repoOutputUsesEndpoint(repoOutput( + "http://[2001:db8::10]:8080/zstack/static/zstack-repo/x86_64", + "http://[2001:db8::10]:8080/zstack/static/qemu-kvm-ev/x86_64"), + "2001:db8::10", 8080) + + assert SshYumRepoChecker.repoOutputUsesEndpoint(repoOutput( + "http://192.168.1.10:8080/zstack/static/zstack-repo/x86_64", + "http://192.168.1.10:8080/zstack/static/qemu-kvm-ev/x86_64"), + "192.168.1.10", 8080) + + assert SshYumRepoChecker.repoOutputUsesEndpoint(repoOutput( + "http://mn.example.com:8080/zstack/static/zstack-repo/x86_64", + "http://mn.example.com:8080/zstack/static/qemu-kvm-ev/x86_64"), + "mn.example.com", 8080) + } + + @Test + void testMismatchMissingAndMalformedRepoEndpoints() { + assert !SshYumRepoChecker.repoOutputUsesEndpoint(repoOutput( + "http://[2001:db8::10]:8080/zstack/static/zstack-repo/x86_64", + "http://192.168.1.10:8080/zstack/static/qemu-kvm-ev/x86_64"), + "2001:db8::10", 8080) + + assert !SshYumRepoChecker.repoOutputUsesEndpoint( + "${ZSTACK_REPO}:baseurl=http://[2001:db8::10]:8080/zstack/static/zstack-repo/x86_64\n", + "2001:db8::10", 8080) + + assert !SshYumRepoChecker.repoOutputUsesEndpoint( + "${ZSTACK_REPO}:baseurl=not-a-url\n${QEMU_REPO}:baseurl=http://[2001:db8::10]:8080/repo\n", + "2001:db8::10", 8080) + } + + private static String repoOutput(String zstackBaseUrl, String qemuBaseUrl) { + return "${ZSTACK_REPO}:baseurl=${zstackBaseUrl}\n${QEMU_REPO}:baseurl=${qemuBaseUrl}\n" + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy b/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy index 2c75a9929af..01c3ca042a3 100644 --- a/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy @@ -95,6 +95,9 @@ class ManagementNetworkIpv6Case extends SubCase { void test() { testSelectManagementServerIpDualStackPolicy() testSelectManagementServerIpSkipsLoopbackAndLinkLocal() + testStrictSelectionWithoutSameFamily() + testStrictSelectionRejectsUnconfiguredRouteSource() + testStrictSelectionReportsRouteFailure() testSelectApplianceVmManagementNodeIpByCidr() testBuildUrlIpv4() testBuildUrlIpv6() @@ -106,6 +109,8 @@ class ManagementNetworkIpv6Case extends SubCase { testKvmAgentUrlsIpv6() testKvmCallbackPrecheckUsesTargetAwareCallbackUrl() testRestFacadeIpv6Urls() + testRestFacadeSelectsSendCommandUrlByTargetIpVersion() + testRestPortFirewallRulesFollowManagementAddressFamilies() testSshTargetUsesRawIpv6Host() testScpTargetUsesBracketedIpv6Host() testCallbackCheckerUsesIpv6Options() @@ -255,7 +260,7 @@ class ManagementNetworkIpv6Case extends SubCase { RESTFacadeImpl.buildCallbackUrl(host, REST_PORT, "zstack") } ] as RESTFacade - String command = KVMHost.buildManagementNodeCallbackCheckCommand(IPV6_2, restf) + String command = KVMHost.buildManagementNodeCallbackCheckCommandForManagementHost(IPV6, restf) String callbackUrl = RESTFacadeImpl.buildCallbackUrl(IPV6, REST_PORT, "zstack") assert callbackUrl == "http://[${IPV6}]:${REST_PORT}/zstack${RESTConstant.CALLBACK_PATH}" @@ -274,6 +279,35 @@ class ManagementNetworkIpv6Case extends SubCase { "http://[2001:db8::1]:8080/zstack${RESTConstant.COMMAND_CHANNEL_PATH}" } + void testRestPortFirewallRulesFollowManagementAddressFamilies() { + List ipv4Rules = [] + List ipv6Rules = [] + + RESTFacadeImpl.installRestPortFirewallRules(REST_PORT, false, + { ipv4Rules.add(it) }, { ipv6Rules.add(it) }) + assert ipv4Rules == ["-A INPUT -p tcp -m state --state NEW -m tcp --dport ${REST_PORT} -j ACCEPT"] + assert ipv6Rules.isEmpty() + + ipv4Rules.clear() + RESTFacadeImpl.installRestPortFirewallRules(REST_PORT, true, + { ipv4Rules.add(it) }, { ipv6Rules.add(it) }) + assert ipv4Rules.size() == 1 + assert ipv6Rules == ipv4Rules + } + + void testRestFacadeSelectsSendCommandUrlByTargetIpVersion() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + String defaultUrl = "http://${IPV4}:${REST_PORT}/zstack${RESTConstant.COMMAND_CHANNEL_PATH}" + + assert RESTFacadeImpl.selectSendCommandUrl(IPV6_2, defaultUrl, REST_PORT, "zstack") == + "http://[${IPV6}]:${REST_PORT}/zstack${RESTConstant.COMMAND_CHANNEL_PATH}" + assert RESTFacadeImpl.selectSendCommandUrl("host.example.com", defaultUrl, REST_PORT, "zstack") == defaultUrl + } + } + void testRestFacadeSelectsCallbackUrlByTargetIpVersion() { withManagementServerIpProperties([ "management.server.ip" : IPV4, @@ -432,6 +466,62 @@ class ManagementNetworkIpv6Case extends SubCase { } } + void testStrictSelectionWithoutSameFamily() { + def selection + withManagementServerIpProperties([ + "management.server.ip": IPV4, + ]) { + selection = Platform.selectManagementServerIpForRemoteStrict( + IPV6_2, IPV6_2, null) + + assert !selection.success + assert selection.failureReason.toString() == "NO_SAME_FAMILY_MGT" + assert selection.selectedIp == null + assert selection.remoteIp == IPV6_2 + assert selection.managementServerIps == [IPV4] + } + assert Platform.managementServerIpSelectionError(selection).globalErrorCode == + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000" + } + + void testStrictSelectionRejectsUnconfiguredRouteSource() { + String unconfiguredRouteSource = "2001:db8::88" + def selection + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + selection = Platform.selectManagementServerIpForRemoteStrict( + IPV6_2, unconfiguredRouteSource, null) + + assert !selection.success + assert selection.failureReason.toString() == "ROUTE_SOURCE_NOT_MANAGEMENT" + assert selection.routeSourceIp == unconfiguredRouteSource + assert selection.managementServerIps == [IPV4, IPV6] + } + assert Platform.managementServerIpSelectionError(selection).globalErrorCode == + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002" + } + + void testStrictSelectionReportsRouteFailure() { + String routeError = "exitCode=2, stderr=network unreachable" + def selection + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + selection = Platform.selectManagementServerIpForRemoteStrict( + IPV6_2, null, routeError) + + assert !selection.success + assert selection.failureReason.toString() == "ROUTE_LOOKUP_FAILED" + assert selection.routeError == routeError + } + def error = Platform.managementServerIpSelectionError(selection) + assert error.globalErrorCode == "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001" + assert error.formatArgs.any { it.contains(routeError) } + } + void testManagementServerSecondaryPropertyRejectsWrongAddressFamily() { withManagementServerIpProperties([ "management.server.ip" : IPV4, diff --git a/test/src/test/groovy/org/zstack/test/integration/core/ansible/AnsibleRunnerTargetAwareArgumentsCase.groovy b/test/src/test/groovy/org/zstack/test/integration/core/ansible/AnsibleRunnerTargetAwareArgumentsCase.groovy new file mode 100644 index 00000000000..a386de35014 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/core/ansible/AnsibleRunnerTargetAwareArgumentsCase.groovy @@ -0,0 +1,256 @@ +package org.zstack.test.integration.core.ansible + +import org.junit.Test +import org.zstack.core.CoreGlobalProperty +import org.zstack.core.Platform +import org.zstack.core.ansible.AnsibleRunner +import org.zstack.core.ansible.CallBackNetworkChecker +import org.zstack.core.ansible.RunAnsibleMsg +import org.zstack.core.cloudbus.CloudBus +import org.zstack.header.core.ReturnValueCompletion +import org.zstack.header.errorcode.ErrorCode +import org.zstack.header.rest.RESTFacade + +import java.lang.reflect.Field +import java.lang.reflect.InvocationHandler +import java.lang.reflect.Method +import java.lang.reflect.Proxy + +class AnsibleRunnerTargetAwareArgumentsCase { + private static final String IPV4 = "127.0.0.1" + private static final String IPV4_TARGET = "127.0.0.2" + private static final String IPV6 = "::1" + private static final String IPV6_TARGET = "::1" + private static final int REST_PORT = 8080 + + @Test + void test() { + String oldUserHome = System.getProperty("user.home") + File testHome = File.createTempDir("zstack-ansible-runner", "") + try { + System.setProperty("user.home", testHome.absolutePath) + testPrimaryIpv4TargetIpv6() + testPrimaryIpv6TargetIpv4() + testHostnameTargetKeepsRestFacadeHost() + testInvalidTargetKeepsRestFacadeHost() + testCallbackCheckerUsesSameSelectedHost() + testNoSameFamilyReturnsErrorWithoutMessage() + } finally { + System.setProperty("user.home", oldUserHome) + testHome.deleteDir() + } + } + + void testPrimaryIpv4TargetIpv6() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + RunAnsibleMsg msg = runAnsible(IPV6_TARGET, IPV4, "http://${IPV4}:${REST_PORT}") + + assert msg.targetIp == IPV6_TARGET + assert msg.deployArguments.pipUrl == "http://[${IPV6}]:${REST_PORT}/zstack/static/pypi/simple" + assert msg.deployArguments.trustedHost == IPV6 + assert msg.deployArguments.yumServer == "[${IPV6}]:${REST_PORT}" + } + } + + void testPrimaryIpv6TargetIpv4() { + withManagementServerIpProperties([ + "management.server.ip" : IPV6, + "management.server.ip4": IPV4, + ]) { + RunAnsibleMsg msg = runAnsible(IPV4_TARGET, IPV6, "http://[${IPV6}]:${REST_PORT}") + + assert msg.targetIp == IPV4_TARGET + assert msg.deployArguments.pipUrl == "http://${IPV4}:${REST_PORT}/zstack/static/pypi/simple" + assert msg.deployArguments.trustedHost == IPV4 + assert msg.deployArguments.yumServer == "${IPV4}:${REST_PORT}" + } + } + + void testHostnameTargetKeepsRestFacadeHost() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + String restHost = "mn.example.com" + RunAnsibleMsg msg = runAnsible("host.example.com", restHost, "http://${restHost}:${REST_PORT}") + + assert msg.deployArguments.pipUrl == "http://${restHost}:${REST_PORT}/zstack/static/pypi/simple" + assert msg.deployArguments.trustedHost == restHost + assert msg.deployArguments.yumServer == "${restHost}:${REST_PORT}" + } + } + + void testInvalidTargetKeepsRestFacadeHost() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + String restHost = "mn.example.com" + RunAnsibleMsg msg = runAnsible("not-an-ip!!", restHost, + "http://${restHost}:${REST_PORT}") + + assert msg.deployArguments.trustedHost == restHost + } + } + + void testNoSameFamilyReturnsErrorWithoutMessage() { + withManagementServerIpProperties([ + "management.server.ip": IPV4, + ]) { + def selection = Platform.selectManagementServerIpForRemoteStrict( + IPV6_TARGET, IPV6_TARGET, null) + def result = runAnsibleResult(IPV6_TARGET, IPV4, + "http://${IPV4}:${REST_PORT}", selection) + + assert result.message == null + assert result.error != null + assert result.error.globalErrorCode == + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000" + } + } + + void testCallbackCheckerUsesSameSelectedHost() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + def result = runAnsibleResult(IPV6_TARGET, IPV4, + "http://${IPV4}:${REST_PORT}", null) + + assert result.error == null + assert result.callbackChecker.callbackIp == IPV6 + } + } + + private RunAnsibleMsg runAnsible(String targetIp, String restHost, String baseUrl) { + def result = runAnsibleResult(targetIp, restHost, baseUrl, null) + assert result.error == null: result.error + assert result.message != null + return result.message as RunAnsibleMsg + } + + private Map runAnsibleResult(String targetIp, String restHost, String baseUrl, def selection) { + RunAnsibleMsg[] captured = new RunAnsibleMsg[1] + ErrorCode[] capturedError = new ErrorCode[1] + CloudBus bus = proxy(CloudBus.class) { Method method, Object[] args -> + if (method.name == "makeTargetServiceIdByResourceUuid") { + return null + } + if (method.name == "send" && args?.length == 2 && args[0] instanceof RunAnsibleMsg) { + captured[0] = args[0] as RunAnsibleMsg + return null + } + return defaultValue(method.returnType) + } + RESTFacade restf = proxy(RESTFacade.class) { Method method, Object[] args -> + switch (method.name) { + case "getBaseUrl": + return baseUrl + case "getHostName": + return restHost + case "getPort": + return REST_PORT + default: + return defaultValue(method.returnType) + } + } + + boolean oldUnitTestOn = CoreGlobalProperty.UNIT_TEST_ON + try { + CoreGlobalProperty.UNIT_TEST_ON = true + AnsibleRunner runner = new AnsibleRunner() + setField(runner, "bus", bus) + setField(runner, "restf", restf) + runner.setForceRun(true) + runner.setTargetIp(targetIp) + runner.setTargetUuid(Platform.uuid) + runner.setPlayBookPath("/tmp/zstack-test-playbook.yaml") + runner.setUsername("root") + runner.setPassword("password") + CallBackNetworkChecker callbackChecker = new CallBackNetworkChecker() + callbackChecker.setCallbackIp("legacy-selection") + runner.installChecker(callbackChecker) + if (selection != null) { + runner.setManagementServerIpSelection(selection) + } + + ReturnValueCompletion completion = new ReturnValueCompletion(null) { + @Override + void success(Boolean returnValue) { + } + + @Override + void fail(ErrorCode errorCode) { + capturedError[0] = errorCode + } + } + runner.run(completion) + + return [message: captured[0], error: capturedError[0], + callbackChecker: callbackChecker] + } finally { + CoreGlobalProperty.UNIT_TEST_ON = oldUnitTestOn + } + } + + private static T proxy(Class type, Closure handler) { + InvocationHandler invocationHandler = { Object ignored, Method method, Object[] args -> + handler.call(method, args) + } as InvocationHandler + return Proxy.newProxyInstance(type.classLoader, [type] as Class[], invocationHandler) as T + } + + private static Object defaultValue(Class returnType) { + if (!returnType.isPrimitive()) { + return null + } + if (returnType == boolean.class) { + return false + } + if (returnType == char.class) { + return (char) 0 + } + return 0 + } + + private static void setField(Object target, String name, Object value) { + Field field = target.class.getDeclaredField(name) + field.setAccessible(true) + field.set(target, value) + } + + private void withManagementServerIpProperties(Map properties, Closure closure) { + List managedKeys = [ + "management.server.ip", + "management.server.ip4", + "management.server.ip6", + ] + Map oldValues = [:] + managedKeys.each { key -> oldValues[key] = System.getProperty(key) } + + try { + resetCachedManagementServerIp() + managedKeys.each { System.clearProperty(it) } + properties.each { key, value -> System.setProperty(key, value) } + closure.call() + } finally { + managedKeys.each { key -> + if (oldValues[key] == null) { + System.clearProperty(key) + } else { + System.setProperty(key, oldValues[key]) + } + } + resetCachedManagementServerIp() + } + } + + private static void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.setAccessible(true) + field.set(null, null) + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/core/salt/SaltDualStackConfigCase.groovy b/test/src/test/groovy/org/zstack/test/integration/core/salt/SaltDualStackConfigCase.groovy new file mode 100644 index 00000000000..883fab17224 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/core/salt/SaltDualStackConfigCase.groovy @@ -0,0 +1,112 @@ +package org.zstack.test.integration.core.salt + +import org.zstack.core.Platform +import org.zstack.core.salt.SaltFacadeImpl +import org.zstack.core.salt.SaltSetupMinionJob +import org.zstack.header.core.ReturnValueCompletion +import org.zstack.header.errorcode.ErrorCode +import org.zstack.testlib.SubCase +import org.zstack.utils.path.PathUtil + +import java.lang.reflect.Field + +class SaltDualStackConfigCase extends SubCase { + @Override + void setup() { + } + + @Override + void environment() { + } + + @Override + void clean() { + } + + @Override + void test() { + testMasterListensOnDualStackSocket() + testMinionSelectsReachableSaltMasterFamily() + testStrictSelectionFailure() + } + + void testMasterListensOnDualStackSocket() { + String masterConfig = PathUtil.findFileOnClassPath("salt/master", true).text + String dualStack = SaltFacadeImpl.renderMasterConfig(masterConfig, 1024, 10, true) + assert dualStack.readLines().contains("interface: '::'") + assert dualStack.readLines().contains("ipv6: true") + + String ipv4Only = SaltFacadeImpl.renderMasterConfig(masterConfig, 1024, 10, false) + assert ipv4Only.readLines().contains("interface: 0.0.0.0") + assert ipv4Only.readLines().contains("ipv6: false") + } + + void testMinionSelectsReachableSaltMasterFamily() { + withManagementServerIpProperties([ + "management.server.ip" : "192.168.1.10", + "management.server.ip6": "2001:db8::10", + ]) { + String template = "master: {managementNodeIp}\nipv6: {ipv6Enabled}\nid: {minionId}\n" + + assert SaltSetupMinionJob.renderMinionConfig(template, "2001:db8::10", "ipv6-minion") == + "master: 2001:db8::10\nipv6: true\nid: ipv6-minion\n" + assert SaltSetupMinionJob.renderMinionConfig(template, "192.168.1.10", "ipv4-minion") == + "master: 192.168.1.10\nipv6: false\nid: ipv4-minion\n" + } + } + + void testStrictSelectionFailure() { + withManagementServerIpProperties([ + "management.server.ip": "127.0.0.1", + ]) { + def selection = Platform.selectManagementServerIpForRemoteStrict( + "::1", "::1", null) + SaltSetupMinionJob job = new SaltSetupMinionJob() + job.setTargetIp("::1") + job.setManagementServerIpSelection(selection) + ErrorCode[] captured = new ErrorCode[1] + + job.run(new ReturnValueCompletion(null) { + @Override + void success(Object ignored) { + assert false: "selection failure must not succeed" + } + + @Override + void fail(ErrorCode errorCode) { + captured[0] = errorCode + } + }) + + assert captured[0] != null + assert captured[0].globalErrorCode == + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000" + } + } + + private static void withManagementServerIpProperties(Map properties, Closure closure) { + List keys = ["management.server.ip", "management.server.ip4", "management.server.ip6"] + Map oldValues = keys.collectEntries { [(it): System.getProperty(it)] } + try { + keys.each { System.clearProperty(it) } + properties.each { key, value -> System.setProperty(key, value) } + resetCachedManagementServerIp() + closure.call() + } finally { + keys.each { key -> + if (oldValues[key] == null) { + System.clearProperty(key) + } else { + System.setProperty(key, oldValues[key]) + } + } + resetCachedManagementServerIp() + } + } + + private static void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.accessible = true + field.set(null, null) + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/host/DualStackAddKvmHostCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/host/DualStackAddKvmHostCase.groovy new file mode 100644 index 00000000000..60d75c8cb57 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/host/DualStackAddKvmHostCase.groovy @@ -0,0 +1,463 @@ +package org.zstack.test.integration.kvm.host + +import org.zstack.core.Platform +import org.zstack.core.CoreGlobalProperty +import org.zstack.core.cloudbus.CloudBus +import org.zstack.core.db.DatabaseFacade +import org.zstack.core.db.Q +import org.zstack.header.host.HostStatus +import org.zstack.header.host.HostConstant +import org.zstack.header.host.PingHostMsg +import org.zstack.header.host.PingHostReply +import org.zstack.header.host.HostVO +import org.zstack.header.host.HostVO_ +import org.zstack.header.rest.RESTFacade +import org.zstack.header.rest.RESTConstant +import org.zstack.header.tag.SystemTagVO +import org.zstack.header.tag.SystemTagVO_ +import org.zstack.header.zone.ZoneVO +import org.zstack.sdk.AddKVMHostAction +import org.zstack.sdk.ClusterInventory +import org.zstack.header.vm.PauseVmOnHypervisorMsg +import org.zstack.header.vm.VmInstanceInventory +import org.zstack.test.integration.kvm.KvmTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import org.zstack.core.ansible.RunAnsibleMsg +import org.zstack.core.ansible.RunAnsibleReply +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KVMConstant +import org.zstack.utils.ShellUtils +import org.zstack.utils.gson.JSONObjectUtil +import org.springframework.http.HttpEntity + +import java.lang.reflect.Field +import java.util.concurrent.TimeUnit + +class DualStackAddKvmHostCase extends SubCase { + private static final String UNIT_TEST_AGENT_HOST = "KVMHost.unitTestAgentHost" + private static final String TEST_INTERFACE = "z86567dummy" + private static final String IPV4_MN = "10.254.86.1" + private static final String IPV4_HOST = "10.254.86.2" + private static final String IPV4_ONLY_HOST = "10.254.86.3" + private static final String IPV6_MN = "fd86:567::1" + private static final String IPV6_MN_ALT = "fd86:567::10" + private static final String IPV6_HOST = "fd86:567::2" + private static final String IPV6_ONLY_HOST = "fd86:567::3" + private static final String IPV6_FAILURE_HOST = "fd86:567::30" + private static final String IPV6_ROUTE_FAILURE_HOST = "fd86:568::2" + + EnvSpec env + ClusterInventory cluster + List oldChronyServers + + @Override + void setup() { + deleteTestInterface() + runNetworkCommand("ip link add ${TEST_INTERFACE} type dummy") + runNetworkCommand("ip link set ${TEST_INTERFACE} up") + runNetworkCommand("ip -4 addr add ${IPV4_MN}/24 dev ${TEST_INTERFACE}") + runNetworkCommand("ip -6 addr add ${IPV6_MN}/64 dev ${TEST_INTERFACE}") + useSpring(KvmTest.springSpec) + } + + @Override + void environment() { + env = HostEnv.noHostBasicEnv() + } + + @Override + void clean() { + System.clearProperty(UNIT_TEST_AGENT_HOST) + CoreGlobalProperty.CHRONY_SERVERS = oldChronyServers + try { + if (env != null) { + env.delete() + } + } finally { + deleteTestInterface() + } + } + + @Override + void test() { + env.create { + cluster = env.inventoryByName("cluster") as ClusterInventory + oldChronyServers = CoreGlobalProperty.CHRONY_SERVERS + try { + System.setProperty(UNIT_TEST_AGENT_HOST, "127.0.0.1") + CoreGlobalProperty.CHRONY_SERVERS = ["127.0.0.1"] + + testPrimaryIpv4AddIpv6Host() + testPrimaryIpv6AddIpv4Host() + testIpv4OnlyRegression() + testIpv6OnlyRegression() + testHostnameRegression() + testPingReResolvesChangedRouteSource() + testPingRouteLookupFailureDoesNotDisconnectHost() + testAnsibleFailureRollsBackHost() + testNoSameFamilyRollsBackWithoutChecker() + testUnconfiguredRouteSourceRollsBackWithoutChecker() + } finally { + System.clearProperty(UNIT_TEST_AGENT_HOST) + CoreGlobalProperty.CHRONY_SERVERS = oldChronyServers + deleteTestInterface() + } + } + } + + void testPrimaryIpv4AddIpv6Host() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4_MN, + "management.server.ip6": IPV6_MN, + ]) { + updateZoneIpVersion("ipv6") + def captured = addHostAndCaptureCommands(IPV6_HOST, "dual-stack-ipv6-host") + RunAnsibleMsg msg = captured.ansible + + assert msg.targetIp == IPV6_HOST + assert msg.deployArguments.pipUrl.contains("[${IPV6_MN}]") + assert msg.deployArguments.trustedHost == IPV6_MN + assert msg.deployArguments.yumServer.startsWith("[${IPV6_MN}]:") + assert captured.connect.sendCommandUrl == bean(RESTFacade.class).buildSendCommandUrl(IPV6_HOST) + assertPingRepairsTargetAwareSendCommandUrl(msg.targetUuid, IPV6_MN) + deleteHost { uuid = msg.targetUuid } + } + } + + void testPrimaryIpv6AddIpv4Host() { + withManagementServerIpProperties([ + "management.server.ip" : IPV6_MN, + "management.server.ip4": IPV4_MN, + ]) { + updateZoneIpVersion("ipv4") + def captured = addHostAndCaptureCommands(IPV4_HOST, "dual-stack-ipv4-host") + RunAnsibleMsg msg = captured.ansible + + assert msg.targetIp == IPV4_HOST + assert msg.deployArguments.pipUrl.contains("//${IPV4_MN}:") + assert msg.deployArguments.trustedHost == IPV4_MN + assert msg.deployArguments.yumServer.startsWith("${IPV4_MN}:") + assert captured.connect.sendCommandUrl == bean(RESTFacade.class).buildSendCommandUrl(IPV4_HOST) + deleteHost { uuid = msg.targetUuid } + } + } + + void testIpv4OnlyRegression() { + withManagementServerIpProperties(["management.server.ip": IPV4_MN]) { + updateZoneIpVersion("ipv4") + def captured = addHostAndCaptureCommands(IPV4_ONLY_HOST, "ipv4-only-host") + RunAnsibleMsg msg = captured.ansible + + assert msg.deployArguments.trustedHost == IPV4_MN + assert captured.connect.sendCommandUrl == bean(RESTFacade.class).buildSendCommandUrl(IPV4_ONLY_HOST) + deleteHost { uuid = msg.targetUuid } + } + } + + void testIpv6OnlyRegression() { + withManagementServerIpProperties(["management.server.ip": IPV6_MN]) { + updateZoneIpVersion("ipv6") + def captured = addHostAndCaptureCommands(IPV6_ONLY_HOST, "ipv6-only-host") + RunAnsibleMsg msg = captured.ansible + + assert msg.deployArguments.trustedHost == IPV6_MN + assert msg.deployArguments.yumServer.startsWith("[${IPV6_MN}]:") + assert captured.connect.sendCommandUrl == bean(RESTFacade.class).buildSendCommandUrl(IPV6_ONLY_HOST) + deleteHost { uuid = msg.targetUuid } + } + } + + void testHostnameRegression() { + withManagementServerIpProperties(["management.server.ip": IPV4_MN]) { + updateZoneIpVersion("ipv4") + def captured = addHostAndCaptureCommands("localhost", "hostname-host") + RunAnsibleMsg msg = captured.ansible + + assert msg.targetIp == "localhost" + assert msg.deployArguments.trustedHost == bean(RESTFacade.class).hostName + assert captured.connect.sendCommandUrl == bean(RESTFacade.class).buildSendCommandUrl("localhost") + deleteHost { uuid = msg.targetUuid } + } + } + + void testPingReResolvesChangedRouteSource() { + withManagementServerIpProperties([ + "management.server.ip" : IPV6_MN, + "management.server.ip6": IPV6_MN_ALT, + ]) { + updateZoneIpVersion("ipv6") + runNetworkCommand("ip -6 addr add ${IPV6_MN_ALT}/64 dev ${TEST_INTERFACE}") + runNetworkCommand("ip -6 route replace ${IPV6_HOST}/128 dev ${TEST_INTERFACE} src ${IPV6_MN}") + try { + def captured = addHostAndCaptureCommands(IPV6_HOST, "route-switch-host") + RunAnsibleMsg msg = captured.ansible + assert msg.deployArguments.trustedHost == IPV6_MN + + runNetworkCommand("ip -6 route replace ${IPV6_HOST}/128 dev ${TEST_INTERFACE} src ${IPV6_MN_ALT}") + assertPingRepairsTargetAwareSendCommandUrl(msg.targetUuid, IPV6_MN_ALT) + assertKvmAsyncCallbackUsesManagementHost(msg.targetUuid, IPV6_MN_ALT) + deleteHost { uuid = msg.targetUuid } + } finally { + ShellUtils.runAndReturn("ip -6 route delete ${IPV6_HOST}/128 dev ${TEST_INTERFACE}", true) + ShellUtils.runAndReturn("ip -6 addr delete ${IPV6_MN_ALT}/64 dev ${TEST_INTERFACE}", true) + } + } + } + + void testAnsibleFailureRollsBackHost() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4_MN, + "management.server.ip6": IPV6_MN, + ]) { + updateZoneIpVersion("ipv6") + String hostUuid = Platform.uuid + boolean ansibleCalled = false + env.message(RunAnsibleMsg.class) { RunAnsibleMsg msg, bus -> + ansibleCalled = true + bus.replyErrorByMessageType(msg, "simulated ansible failure") + } + + def result = addHost(hostUuid, IPV6_FAILURE_HOST, "ansible-failure-host") + + assert ansibleCalled + assert result.error != null + assert !Q.New(HostVO.class).eq(HostVO_.uuid, hostUuid).isExists() + } + } + + void testPingRouteLookupFailureDoesNotDisconnectHost() { + withManagementServerIpProperties([ + "management.server.ip": IPV6_MN, + ]) { + updateZoneIpVersion("ipv6") + runNetworkCommand("ip -6 route replace ${IPV6_ROUTE_FAILURE_HOST}/128 " + + "dev ${TEST_INTERFACE} src ${IPV6_MN}") + try { + def captured = addHostAndCaptureCommands( + IPV6_ROUTE_FAILURE_HOST, "transient-route-failure-host") + String hostUuid = captured.ansible.targetUuid + env.afterSimulator(KVMConstant.KVM_PING_PATH) { + KVMAgentCommands.PingResponse rsp, HttpEntity entity -> + KVMAgentCommands.PingCmd cmd = JSONObjectUtil.toObject( + entity.body, KVMAgentCommands.PingCmd.class) + if (cmd.hostUuid == hostUuid) { + rsp.hostUuid = cmd.hostUuid + rsp.version = bean(DatabaseFacade.class).dbVersion + rsp.sendCommandUrl = "http://stale.example.com/command" + } + return rsp + } + runNetworkCommand("ip -6 route replace unreachable ${IPV6_ROUTE_FAILURE_HOST}/128") + + PingHostMsg msg = new PingHostMsg() + msg.hostUuid = hostUuid + CloudBus bus = bean(CloudBus.class) + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid) + PingHostReply reply = bus.call(msg) as PingHostReply + + assert reply.success + TimeUnit.MILLISECONDS.sleep(500) + HostVO host = Q.New(HostVO.class).eq(HostVO_.uuid, hostUuid).find() + assert host.status == HostStatus.Connected + deleteHost { uuid = hostUuid } + } finally { + ShellUtils.runAndReturn( + "ip -6 route delete ${IPV6_ROUTE_FAILURE_HOST}/128", true) + } + } + } + + void testNoSameFamilyRollsBackWithoutChecker() { + withManagementServerIpProperties([ + "management.server.ip": "127.0.0.1", + ]) { + updateZoneIpVersion("ipv6") + assertSelectionFailureRollsBackWithoutAnsible( + IPV6_HOST, + "no-same-family-host", + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000") + } + } + + void testUnconfiguredRouteSourceRollsBackWithoutChecker() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4_MN, + "management.server.ip6": "fd86:567::10", + ]) { + updateZoneIpVersion("ipv6") + assertSelectionFailureRollsBackWithoutAnsible( + IPV6_HOST, + "unconfigured-route-source-host", + "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002") + } + } + + private void assertSelectionFailureRollsBackWithoutAnsible(String managementIp, + String name, + String globalErrorCode) { + String hostUuid = Platform.uuid + boolean ansibleCalled = false + env.message(RunAnsibleMsg.class) { RunAnsibleMsg msg, bus -> + ansibleCalled = true + bus.reply(msg, new RunAnsibleReply()) + } + + def result = addHost(hostUuid, managementIp, name) + + assert !ansibleCalled + assert result.error != null + assert result.error.globalErrorCode == globalErrorCode + assert !Q.New(HostVO.class).eq(HostVO_.uuid, hostUuid).isExists() + } + + private Map addHostAndCaptureCommands(String managementIp, String name) { + String hostUuid = Platform.uuid + RunAnsibleMsg[] capturedAnsible = new RunAnsibleMsg[1] + KVMAgentCommands.ConnectCmd[] capturedConnect = new KVMAgentCommands.ConnectCmd[1] + env.message(RunAnsibleMsg.class) { RunAnsibleMsg msg, bus -> + capturedAnsible[0] = msg + bus.reply(msg, new RunAnsibleReply()) + } + env.afterSimulator(KVMConstant.KVM_CONNECT_PATH) { rsp, HttpEntity entity -> + KVMAgentCommands.ConnectCmd cmd = JSONObjectUtil.toObject( + entity.body, KVMAgentCommands.ConnectCmd.class) + if (cmd.hostUuid == hostUuid) { + capturedConnect[0] = cmd + } + return rsp + } + + def result = addHost(hostUuid, managementIp, name) + + assert result.error == null: "${result.error?.code} ${result.error?.globalErrorCode} ${result.error?.details}" + assert capturedAnsible[0] != null + assert capturedConnect[0] != null + HostVO host = Q.New(HostVO.class).eq(HostVO_.uuid, hostUuid).find() + assert host != null + assert host.status == HostStatus.Connected + return [ansible: capturedAnsible[0], connect: capturedConnect[0]] + } + + private void assertPingRepairsTargetAwareSendCommandUrl(String hostUuid, String managementHost) { + RESTFacade restf = bean(RESTFacade.class) + DatabaseFacade dbf = bean(DatabaseFacade.class) + KVMAgentCommands.UpdateHostConfigurationCmd[] captured = + new KVMAgentCommands.UpdateHostConfigurationCmd[1] + + env.afterSimulator(KVMConstant.KVM_PING_PATH) { + KVMAgentCommands.PingResponse rsp, HttpEntity entity -> + KVMAgentCommands.PingCmd cmd = JSONObjectUtil.toObject(entity.body, KVMAgentCommands.PingCmd.class) + if (cmd.hostUuid == hostUuid) { + rsp.hostUuid = cmd.hostUuid + rsp.version = dbf.dbVersion + rsp.sendCommandUrl = "http://stale.example.com/command" + } + return rsp + } + env.afterSimulator(KVMConstant.KVM_UPDATE_HOST_CONFIGURATION_PATH) { + KVMAgentCommands.UpdateHostConfigurationResponse rsp, HttpEntity entity -> + KVMAgentCommands.UpdateHostConfigurationCmd cmd = JSONObjectUtil.toObject( + entity.body, KVMAgentCommands.UpdateHostConfigurationCmd.class) + if (cmd.hostUuid == hostUuid) { + captured[0] = cmd + } + return rsp + } + + PingHostMsg msg = new PingHostMsg() + msg.hostUuid = hostUuid + CloudBus bus = bean(CloudBus.class) + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid) + PingHostReply reply = bus.call(msg) as PingHostReply + + assert reply.success + retryInSecs { + assert captured[0] != null + } + assert captured[0].sendCommandUrl == + restf.buildSendCommandUrlForManagementHost(managementHost) + } + + private void assertKvmAsyncCallbackUsesManagementHost(String hostUuid, String managementHost) { + String[] callbackUrl = new String[1] + env.afterSimulator(KVMConstant.KVM_PAUSE_VM_PATH) { rsp, HttpEntity entity -> + callbackUrl[0] = entity.headers.getFirst(RESTConstant.CALLBACK_URL) + return rsp + } + + VmInstanceInventory vm = new VmInstanceInventory() + vm.uuid = Platform.uuid + vm.name = "target-aware-callback-vm" + vm.hostUuid = hostUuid + PauseVmOnHypervisorMsg msg = new PauseVmOnHypervisorMsg() + msg.vmInventory = vm + CloudBus bus = bean(CloudBus.class) + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid) + def reply = bus.call(msg) + + assert reply.success + assert callbackUrl[0] == bean(RESTFacade.class).buildCallbackUrl(managementHost) + } + + private def addHost(String hostUuid, String managementIp, String name) { + AddKVMHostAction action = new AddKVMHostAction() + action.sessionId = adminSession() + action.resourceUuid = hostUuid + action.clusterUuid = cluster.uuid + action.managementIp = managementIp + action.name = name + action.username = "root" + action.password = "password" + return action.call() + } + + private void updateZoneIpVersion(String version) { + SystemTagVO currentTag = Q.New(SystemTagVO.class) + .eq(SystemTagVO_.resourceUuid, cluster.zoneUuid) + .eq(SystemTagVO_.resourceType, ZoneVO.simpleName) + .like(SystemTagVO_.tag, "managementNetwork::ipVersion::%") + .find() + assert currentTag != null + + updateSystemTag { + uuid = currentTag.uuid + tag = "managementNetwork::ipVersion::${version}" + } + } + + private void withManagementServerIpProperties(Map properties, Closure closure) { + List keys = ["management.server.ip", "management.server.ip4", "management.server.ip6"] + Map oldValues = keys.collectEntries { [(it): System.getProperty(it)] } + try { + keys.each { System.clearProperty(it) } + properties.each { key, value -> System.setProperty(key, value) } + resetCachedManagementServerIp() + closure.call() + } finally { + keys.each { key -> + if (oldValues[key] == null) { + System.clearProperty(key) + } else { + System.setProperty(key, oldValues[key]) + } + } + resetCachedManagementServerIp() + } + } + + private static void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.accessible = true + field.set(null, null) + } + + private static void runNetworkCommand(String command) { + def result = ShellUtils.runAndReturn(command, true) + assert result.retCode == 0: "${command}: ${result.stderr}" + } + + private static void deleteTestInterface() { + ShellUtils.runAndReturn("ip link delete ${TEST_INTERFACE}", true) + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/vm/migrate/MigrateVmFromDestitonHostCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/migrate/MigrateVmFromDestitonHostCase.groovy index 5fca7dff6b2..164ea6062ea 100644 --- a/test/src/test/groovy/org/zstack/test/integration/kvm/vm/migrate/MigrateVmFromDestitonHostCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/migrate/MigrateVmFromDestitonHostCase.groovy @@ -2,15 +2,20 @@ package org.zstack.test.integration.kvm.vm.migrate import com.google.gson.Gson import org.springframework.http.HttpEntity +import org.zstack.core.Platform import org.zstack.core.db.Q import org.zstack.core.db.SQL import org.zstack.header.Constants import org.zstack.header.longjob.LongJobState import org.zstack.header.longjob.LongJobVO import org.zstack.header.longjob.LongJobVO_ +import org.zstack.header.host.HostVO +import org.zstack.header.host.HostVO_ import org.zstack.header.vm.APIMigrateVmMsg import org.zstack.header.vm.VmInstanceState import org.zstack.header.vm.VmInstanceVO +import org.zstack.header.rest.RESTConstant +import org.zstack.header.rest.RESTFacade import org.zstack.kvm.KVMAgentCommands import org.zstack.kvm.KVMConstant import org.zstack.kvm.KVMGlobalConfig @@ -24,6 +29,8 @@ import org.zstack.testlib.EnvSpec import org.zstack.testlib.SubCase import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil + +import java.lang.reflect.Field /** * Created by shixin.ruan on 2018/02/10. */ @@ -31,15 +38,21 @@ class MigrateVmFromDestitonHostCase extends SubCase { EnvSpec env VmInstanceInventory vm1 def disconnectHostUuid = [] + Map oldManagementServerIpProperties @Override void clean() { - SQL.New(LongJobVO.class).delete() - env.delete() + try { + SQL.New(LongJobVO.class).delete() + env.delete() + } finally { + restoreManagementServerIpProperties() + } } @Override void setup() { + setManagementServerIpProperties() useSpring(KvmTest.springSpec) spring { ceph() @@ -210,6 +223,11 @@ class MigrateVmFromDestitonHostCase extends SubCase { void testMigrateVmFromDestinationHost() { HostInventory host1 = env.inventoryByName("kvm1") HostInventory host = env.inventoryByName("kvm2") + SQL.New(HostVO.class) + .eq(HostVO_.uuid, host.uuid) + .set(HostVO_.managementIp, "::1") + .update() + host.managementIp = "::1" env.simulator(KVMSecurityGroupBackend.SECURITY_GROUP_CLEANUP_UNUSED_RULE_ON_HOST_PATH) { return new KVMAgentCommands.CleanupUnusedRulesOnHostResponse() @@ -218,8 +236,10 @@ class MigrateVmFromDestitonHostCase extends SubCase { KVMGlobalConfig.MIGRATE_AUTO_CONVERGE.updateValue(true) KVMAgentCommands.MigrateVmCmd cmd = null String huuid = null + String callbackUrl = null env.afterSimulator(KVMConstant.KVM_MIGRATE_VM_PATH) { rsp, HttpEntity entity -> huuid = entity.getHeaders().getFirst(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID) + callbackUrl = entity.headers.getFirst(RESTConstant.CALLBACK_URL) cmd = json(entity.getBody(), KVMAgentCommands.MigrateVmCmd.class) return rsp } @@ -234,6 +254,7 @@ class MigrateVmFromDestitonHostCase extends SubCase { assert cmd.destHostIp == host.managementIp assert cmd.srcHostIp == host1.managementIp assert huuid == host.uuid + assert callbackUrl == bean(RESTFacade.class).buildCallbackUrl(host.managementIp) // confirm migration success VmInstanceVO vo1 @@ -266,6 +287,7 @@ class MigrateVmFromDestitonHostCase extends SubCase { assert !cmd.xbzrle assert cmd.destHostIp == host1.managementIp assert cmd.srcHostIp == host.managementIp + assert callbackUrl == bean(RESTFacade.class).buildCallbackUrl(host.managementIp) retryInSecs { vo1 = dbFindByUuid(vm1.getUuid(), VmInstanceVO.class) @@ -294,6 +316,7 @@ class MigrateVmFromDestitonHostCase extends SubCase { } assert !cmd.autoConverge + assert callbackUrl == bean(RESTFacade.class).buildCallbackUrl(host1.managementIp) } private void migrateUnknownVm(String destHostUuid) { @@ -322,4 +345,33 @@ class MigrateVmFromDestitonHostCase extends SubCase { return Q.New(LongJobVO.class).eq(LongJobVO_.uuid, jobInv.uuid).select(LongJobVO_.state).findValue() == LongJobState.Succeeded } } + + private void setManagementServerIpProperties() { + List keys = ["management.server.ip", "management.server.ip4", "management.server.ip6"] + oldManagementServerIpProperties = keys.collectEntries { [(it): System.getProperty(it)] } + keys.each { System.clearProperty(it) } + System.setProperty("management.server.ip", "127.0.0.1") + System.setProperty("management.server.ip6", "::1") + resetCachedManagementServerIp() + } + + private void restoreManagementServerIpProperties() { + if (oldManagementServerIpProperties == null) { + return + } + oldManagementServerIpProperties.each { key, value -> + if (value == null) { + System.clearProperty(key) + } else { + System.setProperty(key, value) + } + } + resetCachedManagementServerIp() + } + + private static void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.accessible = true + field.set(null, null) + } } diff --git a/utils/src/main/java/org/zstack/utils/IptablesUtils.java b/utils/src/main/java/org/zstack/utils/IptablesUtils.java index fc3e8500b30..94dd7613118 100755 --- a/utils/src/main/java/org/zstack/utils/IptablesUtils.java +++ b/utils/src/main/java/org/zstack/utils/IptablesUtils.java @@ -14,12 +14,20 @@ public static void appendRuleToFilterTable(String rule) { } public static void insertRuleToFilterTable(String rule) { - ShellResult ret = ShellUtils.runAndReturn(String.format("/sbin/iptables-save | grep -- '%s' > /dev/null", rule)); + insertRuleToFilterTable("iptables", rule); + } + + public static void insertRuleToFilterTableV6(String rule) { + insertRuleToFilterTable("ip6tables", rule); + } + + private static void insertRuleToFilterTable(String command, String rule) { + ShellResult ret = ShellUtils.runAndReturn(String.format("/sbin/%s-save | grep -- '%s' > /dev/null", command, rule)); if (ret.getRetCode() == 0) { return; } - ret = ShellUtils.runAndReturn(String.format("/sbin/iptables %s", rule.replace("-A", "-I"))); + ret = ShellUtils.runAndReturn(String.format("/sbin/%s %s", command, rule.replace("-A", "-I"))); ret.raiseExceptionIfFail(); } diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index 9fc175de2c1..cbac287190c 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -1618,6 +1618,12 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_CORE_10001 = "ORG_ZSTACK_CORE_10001"; + public static final String ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000 = "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10000"; + + public static final String ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001 = "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10001"; + + public static final String ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002 = "ORG_ZSTACK_CORE_MANAGEMENT_SERVER_IP_10002"; + public static final String ORG_ZSTACK_VMWARE_CASCADE_10000 = "ORG_ZSTACK_VMWARE_CASCADE_10000"; public static final String ORG_ZSTACK_VMWARE_CASCADE_10001 = "ORG_ZSTACK_VMWARE_CASCADE_10001"; @@ -6785,6 +6791,7 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_BAREMETAL_PXESERVER_10036 = "ORG_ZSTACK_BAREMETAL_PXESERVER_10036"; public static final String ORG_ZSTACK_BAREMETAL_PXESERVER_10037 = "ORG_ZSTACK_BAREMETAL_PXESERVER_10037"; + public static final String ORG_ZSTACK_BAREMETAL_PXESERVER_10038 = "ORG_ZSTACK_BAREMETAL_PXESERVER_10038"; public static final String ORG_ZSTACK_TEST_INTEGRATION_V2V_10000 = "ORG_ZSTACK_TEST_INTEGRATION_V2V_10000";