diff --git a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java index 90aca223f1e..a1e5a0aefe9 100644 --- a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java +++ b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java @@ -140,6 +140,26 @@ public void done(ErrorCodeList errorCodeList) { } }); } + }).then(new NoRollbackFlow() { + String __name__ = "ensure-heartbeat-volume"; + + @Override + public void run(FlowTrigger trigger, Map data) { + ensureHeartbeatVolume(context.getInventory(), extPss, new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList.getCauses().isEmpty()) { + trigger.next(); + } else { + logger.warn(String.format("failed to ensure heartbeat volumes before checking KVM host[uuid:%s, name:%s] storage connection, %s", + context.getInventory().getUuid(), context.getInventory().getName(), errorCodeList.getReadableDetails())); + trigger.fail(operr(new ErrorCodeList().causedBy(errorCodeList.getCauses()), + "failed to ensure heartbeat volumes before checking KVM host[uuid:%s, name:%s] storage connection", + context.getInventory().getUuid(), context.getInventory().getName())); + } + } + }); + } }).then(new NoRollbackFlow() { String __name__ = "check-host-status"; @@ -166,6 +186,25 @@ public void handle(Map data) { }).start(); } + private void ensureHeartbeatVolume(HostInventory host, List extPss, WhileDoneCompletion completion) { + new While<>(extPss).each((extPs, compl) -> { + logger.debug(String.format("ensuring heartbeat volume for external primary storage[uuid:%s, name:%s] before checking KVM host[uuid:%s, name:%s] storage connection", + extPs.getUuid(), extPs.getName(), host.getUuid(), host.getName())); + extPsFactory.getNodeSvc(extPs.getUuid()).activateHeartbeatVolume(host, new ReturnValueCompletion(compl) { + @Override + public void success(HeartbeatVolumeTO returnValue) { + compl.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + compl.addError(errorCode); + compl.done(); + } + }); + }).run(completion); + } + private void deployClient(final KVMHostConnectedContext context, List extPss, WhileDoneCompletion completion) { new While<>(extPss).each((extPs, compl) -> { logger.debug(String.format("deploying client for external primary storage[uuid:%s, name:%s] on KVM host[uuid:%s, name:%s]", diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index f2394a852ba..5a85f0fc4d4 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -28,6 +28,8 @@ import org.zstack.testlib.SubCase import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil +import java.util.concurrent.atomic.AtomicBoolean + /** * @author Xingwei Yu * @date 2024/4/19 10:09 @@ -179,8 +181,25 @@ class ZbsPrimaryStorageCase extends SubCase { clusterUuid = cluster.uuid } + Set heartbeatPools = Collections.synchronizedSet(new HashSet<>()) + AtomicBoolean checkStartedBeforeHeartbeatReady = new AtomicBoolean(false) + env.afterSimulator(ZbsStorageController.CREATE_VOLUME_PATH) { rsp, HttpEntity e -> + ZbsStorageController.CreateVolumeCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CreateVolumeCmd) + if (cmd.volume == ZbsConstants.ZBS_HEARTBEAT_VOLUME_NAME) { + heartbeatPools.add(cmd.logicalPool) + ZbsStorageController.CreateVolumeRsp createVolumeRsp = new ZbsStorageController.CreateVolumeRsp() + createVolumeRsp.installPath = "zbs://${cmd.logicalPool}/${cmd.volume}" + return createVolumeRsp + } + + return rsp + } + env.afterSimulator(ZbsStorageController.CHECK_HOST_STORAGE_CONNECTION_PATH) { rsp, HttpEntity e -> ZbsStorageController.CheckHostStorageConnectionCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CheckHostStorageConnectionCmd) + if (!heartbeatPools.contains("lpool1")) { + checkStartedBeforeHeartbeatReady.set(true) + } ZbsStorageController.CheckHostStorageConnectionRsp checkHostStorageConnectionRsp = new ZbsStorageController.CheckHostStorageConnectionRsp() checkHostStorageConnectionRsp.error = "fake error" @@ -194,6 +213,10 @@ class ZbsPrimaryStorageCase extends SubCase { uuid = kvm.getUuid() } } + retryInSecs { + assert heartbeatPools.contains("lpool1") + assert !checkStartedBeforeHeartbeatReady.get() + } retryInSecs { assert Q.New(PrimaryStorageHostRefVO.class)