Skip to content

Draft: <fix>[zbs]: ensure heartbeat before host check #4533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zstack-robot-2 wants to merge 1 commit into
base: 5.4.12
Choose a base branch
from
Open

Draft: <fix>[zbs]: ensure heartbeat before host check #4533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions ...rc/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,26 @@ public void done(ErrorCodeList errorCodeList) {
}
});
}
}).then(new NoRollbackFlow() {
String __name__ = "ensure-heartbeat-volume";

@Override
public void run(FlowTrigger trigger, Map data) {
ensureHeartbeatVolume(context.getInventory(), extPss, new WhileDoneCompletion(trigger) {
@Override
public void done(ErrorCodeList errorCodeList) {
if (errorCodeList.getCauses().isEmpty()) {
trigger.next();
} else {
logger.warn(String.format("failed to ensure heartbeat volumes before checking KVM host[uuid:%s, name:%s] storage connection, %s",
context.getInventory().getUuid(), context.getInventory().getName(), errorCodeList.getReadableDetails()));
trigger.fail(operr(new ErrorCodeList().causedBy(errorCodeList.getCauses()),
"failed to ensure heartbeat volumes before checking KVM host[uuid:%s, name:%s] storage connection",
context.getInventory().getUuid(), context.getInventory().getName()));
}
}
});
}
}).then(new NoRollbackFlow() {
String __name__ = "check-host-status";

Expand All @@ -166,6 +186,25 @@ public void handle(Map data) {
}).start();
}

private void ensureHeartbeatVolume(HostInventory host, List<ExternalPrimaryStorageVO> extPss, WhileDoneCompletion completion) {
new While<>(extPss).each((extPs, compl) -> {
logger.debug(String.format("ensuring heartbeat volume for external primary storage[uuid:%s, name:%s] before checking KVM host[uuid:%s, name:%s] storage connection",
extPs.getUuid(), extPs.getName(), host.getUuid(), host.getName()));
extPsFactory.getNodeSvc(extPs.getUuid()).activateHeartbeatVolume(host, new ReturnValueCompletion<HeartbeatVolumeTO>(compl) {
@Override
public void success(HeartbeatVolumeTO returnValue) {
compl.done();
}

@Override
public void fail(ErrorCode errorCode) {
compl.addError(errorCode);
compl.done();
}
});
}).run(completion);
}

private void deployClient(final KVMHostConnectedContext context, List<ExternalPrimaryStorageVO> extPss, WhileDoneCompletion completion) {
new While<>(extPss).each((extPs, compl) -> {
logger.debug(String.format("deploying client for external primary storage[uuid:%s, name:%s] on KVM host[uuid:%s, name:%s]",
Expand Down
23 changes: 23 additions & 0 deletions ...groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ import org.zstack.testlib.SubCase
import org.zstack.utils.data.SizeUnit
import org.zstack.utils.gson.JSONObjectUtil

import java.util.concurrent.atomic.AtomicBoolean

/**
* @author Xingwei Yu
* @date 2024/4/19 10:09
Expand Down Expand Up @@ -179,8 +181,25 @@ class ZbsPrimaryStorageCase extends SubCase {
clusterUuid = cluster.uuid
}

Set<String> heartbeatPools = Collections.synchronizedSet(new HashSet<>())
AtomicBoolean checkStartedBeforeHeartbeatReady = new AtomicBoolean(false)
env.afterSimulator(ZbsStorageController.CREATE_VOLUME_PATH) { rsp, HttpEntity<String> e ->
ZbsStorageController.CreateVolumeCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CreateVolumeCmd)
if (cmd.volume == ZbsConstants.ZBS_HEARTBEAT_VOLUME_NAME) {
heartbeatPools.add(cmd.logicalPool)
ZbsStorageController.CreateVolumeRsp createVolumeRsp = new ZbsStorageController.CreateVolumeRsp()
createVolumeRsp.installPath = "zbs://${cmd.logicalPool}/${cmd.volume}"
return createVolumeRsp
}

return rsp
}

env.afterSimulator(ZbsStorageController.CHECK_HOST_STORAGE_CONNECTION_PATH) { rsp, HttpEntity<String> e ->
ZbsStorageController.CheckHostStorageConnectionCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CheckHostStorageConnectionCmd)
if (!heartbeatPools.contains("lpool1")) {
checkStartedBeforeHeartbeatReady.set(true)
}

ZbsStorageController.CheckHostStorageConnectionRsp checkHostStorageConnectionRsp = new ZbsStorageController.CheckHostStorageConnectionRsp()
checkHostStorageConnectionRsp.error = "fake error"
Expand All @@ -194,6 +213,10 @@ class ZbsPrimaryStorageCase extends SubCase {
uuid = kvm.getUuid()
}
}
retryInSecs {
assert heartbeatPools.contains("lpool1")
assert !checkStartedBeforeHeartbeatReady.get()
}

retryInSecs {
assert Q.New(PrimaryStorageHostRefVO.class)
Expand Down