From fc5e3818e11e3182115fc26c4dbfe2477338d431 Mon Sep 17 00:00:00 2001 From: "yingzhe.hu" Date: Tue, 19 May 2026 00:52:45 +0800 Subject: [PATCH 1/3] [ha]: stabilize suspect host pre-fence flow Move premium-only HA pre-fence details out of community while keeping the generic async before-start extension point support. Jira: ZSTAC-83890 Test: mvn -pl header,compute,plugin/kvm,simulator/simulatorImpl,testlib -DskipTests compile Change-Id: I168adf82338f9df9e76287619b7f76a8e5be695f --- .../org/zstack/compute/vm/VmInstanceBase.java | 1 - .../compute/vm/VmStartOnHypervisorFlow.java | 87 ++++++++++++++----- .../header/vm/HaStartVmInstanceMsg.java | 9 ++ ...BeforeStartOnHypervisorExtensionPoint.java | 9 +- 4 files changed, 82 insertions(+), 24 deletions(-) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index ed98aef34e0..d77842b8099 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -8984,4 +8984,3 @@ public void run(MessageReply reply) { }); } } - diff --git a/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java index 20965f61655..bf44adeb836 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java @@ -5,17 +5,26 @@ import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.asyncbatch.While; import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.workflow.FlowChainBuilder; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; import org.zstack.header.core.workflow.Flow; +import org.zstack.header.core.workflow.FlowChain; +import org.zstack.header.core.workflow.FlowDoneHandler; +import org.zstack.header.core.workflow.FlowErrorHandler; import org.zstack.header.core.workflow.FlowRollback; import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorCodeList; import org.zstack.header.host.HostConstant; import org.zstack.header.message.MessageReply; import org.zstack.header.vm.*; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; -import java.util.List; import java.util.Map; @Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) public class VmStartOnHypervisorFlow implements Flow { @@ -26,34 +35,68 @@ public class VmStartOnHypervisorFlow implements Flow { @Autowired private PluginRegistry pluginRgty; - private final List exts = pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class);; - - private void fireExtensions(VmInstanceSpec spec) { - for (VmBeforeStartOnHypervisorExtensionPoint ext : exts) { - ext.beforeStartVmOnHypervisor(spec); - } - } - @Override public void run(final FlowTrigger chain, final Map data) { final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString()); + FlowChain fchain = FlowChainBuilder.newSimpleFlowChain(); + fchain.setName(String.format("vm-start-on-hypervisor-vm-%s", spec.getVmInventory().getUuid())); + fchain.then(new NoRollbackFlow() { + @Override + public void run(FlowTrigger trigger, Map d) { + new While<>(pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class)) + .each((ext, comp) -> ext.beforeStartVmOnHypervisor(spec, new Completion(comp) { + @Override + public void success() { + comp.done(); + } - fireExtensions(spec); - - StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg(); - msg.setVmSpec(spec); - bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid()); - bus.send(msg, new CloudBusCallBack(chain) { + @Override + public void fail(ErrorCode errorCode) { + comp.addError(errorCode); + comp.done(); + } + })).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList.getCauses().size() > 0) { + trigger.fail(errorCodeList.getCauses().get(0)); + } else { + trigger.next(); + } + } + }); + } + }); + fchain.then(new NoRollbackFlow() { @Override - public void run(MessageReply reply) { - if (reply.isSuccess()) { - data.put(VmStartOnHypervisorFlow.class.getName(), true); - chain.next(); - } else { - chain.fail(reply.getError()); - } + public void run(FlowTrigger trigger, Map d) { + StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg(); + msg.setVmSpec(spec); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid()); + bus.send(msg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + data.put(VmStartOnHypervisorFlow.class.getName(), true); + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); } }); + fchain.done(new FlowDoneHandler(chain) { + @Override + public void handle(Map d) { + chain.next(); + } + }).error(new FlowErrorHandler(chain) { + @Override + public void handle(ErrorCode errCode, Map d) { + chain.fail(errCode); + } + }).start(); } @Override diff --git a/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java b/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java index a358249cdf9..3ef49037137 100644 --- a/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java +++ b/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java @@ -12,6 +12,7 @@ public class HaStartVmInstanceMsg extends NeedReplyMessage implements VmInstance private String vmInstanceUuid; private String judgerClassName; private List softAvoidHostUuids; + private String accessiblePeerHostUuid; private String haReason; public String getJudgerClassName() { @@ -30,6 +31,14 @@ public void setSoftAvoidHostUuids(List softAvoidHostUuids) { this.softAvoidHostUuids = softAvoidHostUuids; } + public String getAccessiblePeerHostUuid() { + return accessiblePeerHostUuid; + } + + public void setAccessiblePeerHostUuid(String accessiblePeerHostUuid) { + this.accessiblePeerHostUuid = accessiblePeerHostUuid; + } + @Override public String getVmInstanceUuid() { return vmInstanceUuid; diff --git a/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java b/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java index c56d99d38dd..1b3a393496f 100755 --- a/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java +++ b/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java @@ -1,7 +1,14 @@ package org.zstack.header.vm; +import org.zstack.header.core.Completion; + /** */ public interface VmBeforeStartOnHypervisorExtensionPoint { - void beforeStartVmOnHypervisor(VmInstanceSpec spec); + default void beforeStartVmOnHypervisor(VmInstanceSpec spec) {} + + default void beforeStartVmOnHypervisor(VmInstanceSpec spec, Completion completion) { + beforeStartVmOnHypervisor(spec); + completion.success(); + } } From d9911551f11be8b512d24e09cb93bee65c257e39 Mon Sep 17 00:00:00 2001 From: "yingzhe.hu" Date: Wed, 20 May 2026 16:57:11 +0800 Subject: [PATCH 2/3] [ha]: create pre-fence tag in core Move HA pre-fence pending tag creation into HaStartVmInstanceMsg handling after the VM state update, so later pre-fence execution observes the same database state. Resolves: ZSTAC-83890 Change-Id: I7f046732e0245319ed4cf188d1342b65dad11767 --- .../org/zstack/compute/vm/VmInstanceBase.java | 24 +++++++++++++++++++ .../org/zstack/compute/vm/VmSystemTags.java | 7 ++++++ 2 files changed, 31 insertions(+) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index d77842b8099..7232d478680 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -1014,6 +1014,8 @@ public void done(ErrorCodeList errorCodeList) { } sql.update(); + refreshVO(); + createPreFencePendingTag(msg); startVm(msg, new Completion(msg, chain) { @Override @@ -1044,6 +1046,28 @@ public String getName() { }); } + private void createPreFencePendingTag(HaStartVmInstanceMsg msg) { + String suspectHostUuid = StringUtils.trimToNull(CollectionUtils.isEmpty(msg.getSoftAvoidHostUuids()) ? + self.getLastHostUuid() : msg.getSoftAvoidHostUuids().get(0)); + String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); + if (suspectHostUuid == null || peerHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host[%s] or peer host[%s] is absent", + self.getUuid(), suspectHostUuid, peerHostUuid)); + return; + } + + Map tokens = new HashMap<>(); + tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); + tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); + + SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); + creator.inherent = true; + creator.recreate = true; + creator.ignoreIfExisting = true; + creator.setTagByTokens(tokens); + creator.create(); + } + private void changeVmIp(final String l3Uuid, final Map staticIpMap, final Completion completion) { final VmNicVO targetNic = CollectionUtils.find(self.getVmNics(), new Function() { @Override diff --git a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java index 21c9668175a..ebb34e5970f 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java @@ -309,5 +309,12 @@ public String desensitizeTag(SystemTag systemTag, String tag) { public static PatternedSystemTag VM_STATE_PAUSED_AFTER_MIGRATE = new PatternedSystemTag(("vmPausedAfterMigrate"), VmInstanceVO.class); + public static String HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN = "suspectHostUuid"; + public static String HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN = "accessiblePeerHostUuid"; + public static PatternedSystemTag HA_PRE_FENCE_PENDING = + new PatternedSystemTag(String.format("haPreFencePending::{%s}::{%s}", + HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, + HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN), VmInstanceVO.class); + public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); } From 118e53d14a81ca48fa32af00b69aca7c51b7b7d0 Mon Sep 17 00:00:00 2001 From: "yingzhe.hu" Date: Thu, 28 May 2026 15:43:20 +0800 Subject: [PATCH 3/3] [ha]: require peer for pre-fence tag Create HA pre-fence pending tag only when the checker provides an accessible peer host. The peer should be covered by the HA dead-host and storage-inactive paths, so do not persist a fallback marker when it is absent. Resolves: ZSTAC-83890 Change-Id: I36426b3386f1080ddeae8396ea9c4f7a81e25711 --- .../org/zstack/compute/vm/VmInstanceBase.java | 48 +++++++++---------- .../org/zstack/compute/vm/VmSystemTags.java | 4 +- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index 7232d478680..9e03bfa61b3 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -1004,18 +1004,38 @@ public void done(ErrorCodeList errorCodeList) { logger.debug(String.format("HaStartVmJudger[%s] says the VM[uuid:%s, name:%s] is qualified for HA start, now we are starting it", judger.getClass(), self.getUuid(), self.getName())); + String hostUuid = self.getHostUuid(); + String suspectHostUuid = StringUtils.trimToNull(hostUuid); + String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); UpdateQuery sql = SQL.New(VmInstanceVO.class) .eq(VmInstanceVO_.uuid, self.getUuid()) .set(VmInstanceVO_.state, VmInstanceState.Stopped) .set(VmInstanceVO_.hostUuid, null); - if (self.getHostUuid() != null) { - sql.set(VmInstanceVO_.lastHostUuid, self.getHostUuid()); + if (hostUuid != null) { + sql.set(VmInstanceVO_.lastHostUuid, hostUuid); } sql.update(); refreshVO(); - createPreFencePendingTag(msg); + if (suspectHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host is absent", + self.getUuid())); + } else if (peerHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because peer host is absent", + self.getUuid())); + } else { + Map tokens = new HashMap<>(); + tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); + tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); + + SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); + creator.inherent = true; + creator.recreate = true; + creator.ignoreIfExisting = true; + creator.setTagByTokens(tokens); + creator.create(); + } startVm(msg, new Completion(msg, chain) { @Override @@ -1046,28 +1066,6 @@ public String getName() { }); } - private void createPreFencePendingTag(HaStartVmInstanceMsg msg) { - String suspectHostUuid = StringUtils.trimToNull(CollectionUtils.isEmpty(msg.getSoftAvoidHostUuids()) ? - self.getLastHostUuid() : msg.getSoftAvoidHostUuids().get(0)); - String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); - if (suspectHostUuid == null || peerHostUuid == null) { - logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host[%s] or peer host[%s] is absent", - self.getUuid(), suspectHostUuid, peerHostUuid)); - return; - } - - Map tokens = new HashMap<>(); - tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); - tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); - - SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); - creator.inherent = true; - creator.recreate = true; - creator.ignoreIfExisting = true; - creator.setTagByTokens(tokens); - creator.create(); - } - private void changeVmIp(final String l3Uuid, final Map staticIpMap, final Completion completion) { final VmNicVO targetNic = CollectionUtils.find(self.getVmNics(), new Function() { @Override diff --git a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java index ebb34e5970f..6a146649119 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java @@ -309,12 +309,12 @@ public String desensitizeTag(SystemTag systemTag, String tag) { public static PatternedSystemTag VM_STATE_PAUSED_AFTER_MIGRATE = new PatternedSystemTag(("vmPausedAfterMigrate"), VmInstanceVO.class); + public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); + public static String HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN = "suspectHostUuid"; public static String HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN = "accessiblePeerHostUuid"; public static PatternedSystemTag HA_PRE_FENCE_PENDING = new PatternedSystemTag(String.format("haPreFencePending::{%s}::{%s}", HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN), VmInstanceVO.class); - - public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); }