diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 34e0c434296..57ad97b70e3 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -5270,6 +5270,15 @@ private boolean noStorageAccessible(){ return inaccessiblePsCount == attachedPsCount && attachedPsCount > 0; } + @Transactional(readOnly = true) + private long getLibvirtRestartedEchoTimeout() { + long vmCount = KVMHostUtils.countVmsForLibvirtRestartEchoTimeout(self.getUuid()); + long timeout = KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(vmCount); + logger.info(String.format("extend kvmagent echo timeout after libvirtd restart on host[uuid:%s, vmCount:%s] to %sms", + self.getUuid(), vmCount, timeout)); + return timeout; + } + private void updateHostOsInformation(String distro, String release, String version) { final KVMHostVO kvmHostVO = getSelf(); kvmHostVO.setOsDistribution(distro); @@ -5540,6 +5549,7 @@ public void connectHook(final ConnectHostInfo info, final Completion complete) { chain.allowWatch(); chain.then(new ShareFlow() { boolean deployed = false; + boolean libvirtRestarted = false; @Override public void setup() { flow(new NoRollbackFlow() { @@ -5814,6 +5824,8 @@ public void success(Boolean run) { deployed = run; } if (deployed) { + libvirtRestarted = KVMHostUtils.shouldRestartLibvirtdDuringDeploy( + deployArguments.getInit(), deployArguments.getRestartLibvirtd()); // update host agent version when open grayScaleUpgrade upgradeChecker.updateAgentVersion(self.getUuid(), AnsibleConstant.KVM_AGENT_NAME, dbf.getDbVersion(), dbf.getDbVersion()); } @@ -5888,6 +5900,9 @@ public boolean skip(Map data) { @Override public void run(final FlowTrigger trigger, Map data) { + final long echoTimeout = libvirtRestarted + ? getLibvirtRestartedEchoTimeout() + : TimeUnit.SECONDS.toMillis(CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT); restf.echo(echoPath, new Completion(trigger) { @Override public void success() { @@ -5919,7 +5934,7 @@ public void success() { public void fail(ErrorCode errorCode) { trigger.fail(errorCode); } - }); + }, TimeUnit.SECONDS.toMillis(1), echoTimeout); } @Override @@ -5931,7 +5946,7 @@ public void fail(ErrorCode errorCode) { trigger.fail(errorCode); } } - }); + }, TimeUnit.SECONDS.toMillis(1), echoTimeout); } }); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java index 3b9979caeb3..cafee683c5e 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java @@ -1,12 +1,16 @@ package org.zstack.kvm; import org.apache.commons.codec.digest.DigestUtils; +import org.zstack.core.CoreGlobalProperty; import org.zstack.core.db.Q; import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.network.l2.*; import org.zstack.header.tag.SystemTagVO; import org.zstack.header.tag.SystemTagVO_; import org.zstack.header.tag.TagType; +import org.zstack.header.vm.VmInstanceState; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; import org.zstack.utils.TagUtils; import org.zstack.utils.logging.CLogger; import org.zstack.utils.logging.CLoggerImpl; @@ -16,6 +20,7 @@ import java.util.List; import java.util.Locale; import java.util.Set; +import java.util.concurrent.TimeUnit; /** * Created by GuoYi on 4/16/20. @@ -23,6 +28,33 @@ public class KVMHostUtils { private static final CLogger logger = CLoggerImpl.getLogger(KVMHostUtils.class); + public static final long LIBVIRT_RESTART_ECHO_TIMEOUT_VM_THRESHOLD = 100; + public static final long LIBVIRT_RESTART_ECHO_TIMEOUT_PER_VM_SECONDS = 1; + public static final long LIBVIRT_RESTART_ECHO_TIMEOUT_MAX_SECONDS = 180; + + public static boolean shouldRestartLibvirtdDuringDeploy(String init, String restartLibvirtd) { + return "true".equalsIgnoreCase(init) || "true".equalsIgnoreCase(restartLibvirtd); + } + + public static long countVmsForLibvirtRestartEchoTimeout(String hostUuid) { + return Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.hostUuid, hostUuid) + .notEq(VmInstanceVO_.state, VmInstanceState.Stopped) + .count(); + } + + public static long calculateLibvirtRestartEchoTimeoutMillis(long vmCount) { + long defaultTimeoutSeconds = CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT; + if (vmCount <= LIBVIRT_RESTART_ECHO_TIMEOUT_VM_THRESHOLD) { + return TimeUnit.SECONDS.toMillis(defaultTimeoutSeconds); + } + + long maxExtraSeconds = Math.max(0, LIBVIRT_RESTART_ECHO_TIMEOUT_MAX_SECONDS - defaultTimeoutSeconds); + long extraVmCount = vmCount - LIBVIRT_RESTART_ECHO_TIMEOUT_VM_THRESHOLD; + long extraSeconds = Math.min(extraVmCount * LIBVIRT_RESTART_ECHO_TIMEOUT_PER_VM_SECONDS, maxExtraSeconds); + return TimeUnit.SECONDS.toMillis(defaultTimeoutSeconds + extraSeconds); + } + public static boolean shouldContinueReconnectOnAnsibleFailure(boolean isNewAdded, ErrorCode errorCode) { return !isNewAdded && isLibvirtSocketMaskSystemdTimeout(errorCode); } diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/host/LibvirtRestartEchoTimeoutCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/host/LibvirtRestartEchoTimeoutCase.groovy new file mode 100644 index 00000000000..b67eced4a13 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/host/LibvirtRestartEchoTimeoutCase.groovy @@ -0,0 +1,86 @@ +package org.zstack.test.integration.kvm.host + +import org.zstack.core.CoreGlobalProperty +import org.zstack.core.db.Q +import org.zstack.core.db.SQL +import org.zstack.header.vm.VmInstanceState +import org.zstack.header.vm.VmInstanceVO +import org.zstack.header.vm.VmInstanceVO_ +import org.zstack.kvm.KVMHostUtils +import org.zstack.sdk.HostInventory +import org.zstack.sdk.VmInstanceInventory +import org.zstack.test.integration.kvm.Env +import org.zstack.test.integration.kvm.KvmTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase + +import java.util.concurrent.TimeUnit + +class LibvirtRestartEchoTimeoutCase extends SubCase { + EnvSpec env + + @Override + void setup() { + useSpring(KvmTest.springSpec) + } + + @Override + void environment() { + env = Env.oneVmBasicEnv() + } + + @Override + void test() { + env.create { + testCalculateLibvirtRestartEchoTimeout() + testCountVmsForLibvirtRestartEchoTimeoutExcludesStoppedVm() + } + } + + @Override + void clean() { + env.delete() + } + + void testCalculateLibvirtRestartEchoTimeout() { + int oldTimeout = CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT + try { + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = 60 + + assert KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(100) == TimeUnit.SECONDS.toMillis(60) + assert KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(101) == TimeUnit.SECONDS.toMillis(61) + assert KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(10000) == TimeUnit.SECONDS.toMillis(180) + + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = 300 + assert KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(10000) == TimeUnit.SECONDS.toMillis(300) + } finally { + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = oldTimeout + } + } + + void testCountVmsForLibvirtRestartEchoTimeoutExcludesStoppedVm() { + HostInventory host = env.inventoryByName("kvm") as HostInventory + VmInstanceInventory vm = env.inventoryByName("vm") as VmInstanceInventory + + long originalCount = KVMHostUtils.countVmsForLibvirtRestartEchoTimeout(host.uuid) + assert originalCount > 0 + + VmInstanceState originalState = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.state) + .eq(VmInstanceVO_.uuid, vm.uuid) + .findValue() + try { + SQL.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vm.uuid) + .set(VmInstanceVO_.state, VmInstanceState.Stopped) + .update() + + assert KVMHostUtils.countVmsForLibvirtRestartEchoTimeout(host.uuid) == originalCount - 1 + } finally { + SQL.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vm.uuid) + .set(VmInstanceVO_.state, originalState) + .update() + } + } +} diff --git a/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java b/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java index 2e0e06c7dfc..d0f38d61641 100644 --- a/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java +++ b/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java @@ -2,10 +2,75 @@ import org.junit.Assert; import org.junit.Test; +import org.zstack.core.CoreGlobalProperty; import org.zstack.header.errorcode.ErrorCode; import org.zstack.kvm.KVMHostUtils; +import java.util.concurrent.TimeUnit; + public class KVMHostUtilsTest { + @Test + public void shouldRestartLibvirtdDuringDeploy_initOrRestartLibvirtdTriggers() { + Assert.assertFalse(KVMHostUtils.shouldRestartLibvirtdDuringDeploy(null, null)); + Assert.assertFalse(KVMHostUtils.shouldRestartLibvirtdDuringDeploy("false", "false")); + Assert.assertTrue(KVMHostUtils.shouldRestartLibvirtdDuringDeploy("true", "false")); + Assert.assertTrue(KVMHostUtils.shouldRestartLibvirtdDuringDeploy("false", "true")); + Assert.assertTrue(KVMHostUtils.shouldRestartLibvirtdDuringDeploy("TrUe", null)); + Assert.assertTrue(KVMHostUtils.shouldRestartLibvirtdDuringDeploy(null, "TrUe")); + } + + @Test + public void calculateLibvirtRestartEchoTimeout_keepsDefaultAtThreshold() { + int oldTimeout = CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT; + try { + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = 60; + Assert.assertEquals(TimeUnit.SECONDS.toMillis(60), + KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(100)); + } finally { + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = oldTimeout; + } + } + + @Test + public void calculateLibvirtRestartEchoTimeout_addsOneSecondAfterThreshold() { + int oldTimeout = CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT; + try { + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = 60; + Assert.assertEquals(TimeUnit.SECONDS.toMillis(61), + KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(101)); + } finally { + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = oldTimeout; + } + } + + @Test + public void calculateLibvirtRestartEchoTimeout_capsAt180Seconds() { + int oldTimeout = CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT; + try { + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = 60; + Assert.assertEquals(TimeUnit.SECONDS.toMillis(180), + KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(10000)); + } finally { + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = oldTimeout; + } + } + + @Test + public void calculateLibvirtRestartEchoTimeout_doesNotReduceConfiguredTimeoutAboveCap() { + int oldTimeout = CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT; + try { + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = 180; + Assert.assertEquals(TimeUnit.SECONDS.toMillis(180), + KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(10000)); + + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = 300; + Assert.assertEquals(TimeUnit.SECONDS.toMillis(300), + KVMHostUtils.calculateLibvirtRestartEchoTimeoutMillis(10000)); + } finally { + CoreGlobalProperty.REST_FACADE_ECHO_TIMEOUT = oldTimeout; + } + } + @Test public void zstac86349_continueReconnectOnLibvirtSocketMaskSystemdTimeout() { ErrorCode error = new ErrorCode();