From 928aa8e6fd1ae669d37e020f74e052035589d8a8 Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Mon, 7 Sep 2026 16:35:54 +0900 Subject: [PATCH 1/2] =?UTF-8?q?=EB=B0=B1=EC=97=85=20=EC=9C=A0=EC=A7=80=20?= =?UTF-8?q?=EA=B0=9C=EC=88=98=20=EC=9E=90=EB=8F=99=20=EC=82=AD=EC=A0=9C=20?= =?UTF-8?q?timeout=20=EC=B2=98=EB=A6=AC=20=EA=B0=9C=EC=84=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Ablestack 백업 삭제 명령에 backup.command.timeout 적용 - KVM Agent에서 Ablestack 백업 삭제 스크립트 실행 시 command wait timeout 반영 - 스케줄 백업 생성 후 유지 개수 자동 삭제 실패 시 백업 생성 작업은 실패 처리되지 않도록 분리 - Ablestack Commvault, NetBackup artifact cleanup 명령에도 동일한 timeout 처리 반영 --- .../backup/AblestackCommvaultBackupProvider.java | 4 ++++ .../cloudstack/backup/AblestackNasBackupProvider.java | 4 ++++ .../cloudstack/backup/AblestackNetBackupProvider.java | 8 ++++++++ .../LibvirtAblestackDeleteBackupCommandWrapper.java | 5 ++++- .../org/apache/cloudstack/backup/BackupManagerImpl.java | 7 ++++++- 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/plugins/backup/ablestack-commvault/src/main/java/org/apache/cloudstack/backup/AblestackCommvaultBackupProvider.java b/plugins/backup/ablestack-commvault/src/main/java/org/apache/cloudstack/backup/AblestackCommvaultBackupProvider.java index d9452f37bf8d..8346e04cb6a5 100644 --- a/plugins/backup/ablestack-commvault/src/main/java/org/apache/cloudstack/backup/AblestackCommvaultBackupProvider.java +++ b/plugins/backup/ablestack-commvault/src/main/java/org/apache/cloudstack/backup/AblestackCommvaultBackupProvider.java @@ -2627,6 +2627,10 @@ private void cleanupBackupPathOnStageHost(String clientName, String path, boolea throw new CloudRuntimeException(String.format("Unable to find stage host [%s] for backup cleanup", clientName)); } AblestackDeleteBackupCommand command = new AblestackDeleteBackupCommand(path, null, null, null, forced); + final int commandTimeout = BackupCommandTimeout.value(); + if (commandTimeout > 0) { + command.setWait(commandTimeout); + } command.setBackupProvider("ablestack-commvault"); command.setVmName(vmName); command.setCheckpointName(checkpointName); diff --git a/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java b/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java index 6c3ed15bcb48..9d400ce1d134 100644 --- a/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java +++ b/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java @@ -1192,6 +1192,10 @@ public boolean deleteBackup(Backup backup, boolean forced) { AblestackDeleteBackupCommand command = new AblestackDeleteBackupCommand(backup.getExternalId(), backupRepository.getType(), backupRepository.getAddress(), backupRepository.getMountOptions(), forced); + final int commandTimeout = BackupCommandTimeout.value(); + if (commandTimeout > 0) { + command.setWait(commandTimeout); + } command.setMountTimeout(NASBackupRestoreMountTimeout.value()); command.setBackupProvider("ablestack-nas"); command.setVmName(vm != null ? vm.getInstanceName() : null); diff --git a/plugins/backup/ablestack-netbackup/src/main/java/org/apache/cloudstack/backup/AblestackNetBackupProvider.java b/plugins/backup/ablestack-netbackup/src/main/java/org/apache/cloudstack/backup/AblestackNetBackupProvider.java index cf1b6e9ef4a4..597434ca922c 100644 --- a/plugins/backup/ablestack-netbackup/src/main/java/org/apache/cloudstack/backup/AblestackNetBackupProvider.java +++ b/plugins/backup/ablestack-netbackup/src/main/java/org/apache/cloudstack/backup/AblestackNetBackupProvider.java @@ -497,6 +497,10 @@ private boolean cleanupFailedBackupArtifacts(final Host host, final Backup backu && StringUtils.isNotBlank(getBackupDetail(backup, DETAIL_CHECKPOINT_NAME)) && StringUtils.isNotBlank(getBackupDetail(backup, DETAIL_RBD_DISK_PATHS))) { final AblestackDeleteBackupCommand command = new AblestackDeleteBackupCommand(backup.getExternalId(), null, null, null, true); + final int commandTimeout = BackupCommandTimeout.value(); + if (commandTimeout > 0) { + command.setWait(commandTimeout); + } command.setBackupProvider(getName()); final VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); command.setVmName(vm != null ? vm.getInstanceName() : null); @@ -2097,6 +2101,10 @@ private void cleanupExpiredBackupArtifact(final Backup backup, final Set b } final AblestackDeleteBackupCommand command = new AblestackDeleteBackupCommand(backup.getExternalId(), null, null, null, true); + final int commandTimeout = BackupCommandTimeout.value(); + if (commandTimeout > 0) { + command.setWait(commandTimeout); + } command.setBackupProvider(getName()); final VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); command.setVmName(vm != null ? vm.getInstanceName() : null); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackDeleteBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackDeleteBackupCommandWrapper.java index e68ea79f773d..38d7bf4b1ebb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackDeleteBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackDeleteBackupCommandWrapper.java @@ -46,6 +46,7 @@ public Answer execute(AblestackDeleteBackupCommand command, LibvirtComputingReso final String cleanupCheckpointNames = command.getCleanupCheckpointNames(); final String diskPaths = command.getDiskPaths(); final boolean forced = command.isForced(); + int timeout = command.getWait() > 0 ? command.getWait() * 1000 : libvirtComputingResource.getCmdsTimeout(); List commands = new ArrayList<>(); if ("ablestack-commvault".equalsIgnoreCase(backupProvider)) { @@ -136,7 +137,9 @@ public Answer execute(AblestackDeleteBackupCommand command, LibvirtComputingReso commands.add(deleteCommand.toArray(new String[0])); } - Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + logger.debug("Starting Ablestack backup delete for provider [{}], path [{}] with timeout [{}] ms", + backupProvider, backupPath, timeout); + Pair result = Script.executePipedCommands(commands, timeout); logger.debug(String.format("Backup delete result: %s , exit code: %s", result.second(), result.first())); diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index b384f295427d..bf23f5214011 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -1034,7 +1034,12 @@ public boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllo } createCheckedBackup(cmd, owner, isScheduledBackup, backupSize, vm, vmId, backupProvider, backupScheduleId); if (isScheduledBackup) { - deleteOldestBackupFromScheduleIfRequired(vmId, backupScheduleId); + try { + deleteOldestBackupFromScheduleIfRequired(vmId, backupScheduleId); + } catch (RuntimeException e) { + logger.warn("Failed to apply backup retention cleanup after creating scheduled backup for VM [ID: {}], schedule [ID: {}]. " + + "The backup creation flow will not be failed by this cleanup error.", vmId, backupScheduleId, e); + } } logger.info("Completed VM backup request [vmId: {}, vmUuid: {}, vmName: {}, provider: {}, offeringId: {}, scheduleId: {}, elapsedMs: {}]", vm.getId(), vm.getUuid(), vm.getInstanceName(), offering.getProvider(), offering.getId(), backupScheduleId, System.currentTimeMillis() - backupStartTime); From 51ef9dc12f9d6951063bb9ae4babb4395dc436d3 Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Mon, 7 Sep 2026 16:48:34 +0900 Subject: [PATCH 2/2] =?UTF-8?q?=EB=B0=B1=EC=97=85=20=EC=82=AD=EC=A0=9C=20?= =?UTF-8?q?=EC=8B=A4=ED=8C=A8=EB=90=9C=20=EB=B0=B1=EC=97=85=EB=B3=B8=20det?= =?UTF-8?q?ails=EC=97=90=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cloudstack/backup/BackupManagerImpl.java | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index bf23f5214011..8af0d8e4076c 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -280,6 +280,10 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { private static final String FAST_CLONE_FLATTEN_STATUS = "clone.fast.flatten.status"; private static final String FAST_CLONE_FLATTEN_PENDING = "pending"; private static final String FAST_CLONE_FLATTEN_RUNNING = "running"; + private static final String RETENTION_CLEANUP_FAILED = "retention.cleanup.failed"; + private static final String RETENTION_CLEANUP_FAILED_AT = "retention.cleanup.failed.at"; + private static final String RETENTION_CLEANUP_SCHEDULE_ID = "retention.cleanup.schedule.id"; + private static final String RETENTION_CLEANUP_REASON = "retention.cleanup.reason"; private static Map backupProvidersMap = new HashMap<>(); private static final String ABLESTACK_NETBACKUP_PROVIDER_NAME = "ablestack-netbackup"; @@ -1350,14 +1354,18 @@ private boolean deleteBackupChain(List chain, long backupScheduleId) { while (!remainingBackups.isEmpty()) { List leafBackups = getLeafBackups(remainingBackups); if (CollectionUtils.isEmpty(leafBackups)) { - logger.warn("Could not find a deletable leaf while removing an obsolete backup chain for {}.", cleanupTarget); + String reason = "Could not find a deletable leaf while removing an obsolete backup chain"; + logger.warn("{} for {}.", reason, cleanupTarget); + markRetentionCleanupFailure(remainingBackups, backupScheduleId, reason); return false; } for (BackupVO backup : leafBackups) { try { if (!deleteBackup(backup.getId(), false)) { + String reason = "deleteBackup returned false"; logger.warn("Failed to delete backup [ID: {}, UUID: {}] while deleting a chain for {}.", backup.getId(), backup.getUuid(), cleanupTarget); + markRetentionCleanupFailure(remainingBackups, backupScheduleId, reason); return false; } String eventDescription = backupScheduleId > 0 @@ -1373,6 +1381,7 @@ private boolean deleteBackupChain(List chain, long backupScheduleId) { } catch (Exception e) { logger.warn("Skipping retention deletion for backup [ID: {}, UUID: {}] on {} because it is not currently safe to remove: {}", backup.getId(), backup.getUuid(), cleanupTarget, e.getMessage()); + markRetentionCleanupFailure(remainingBackups, backupScheduleId, e.getMessage()); return false; } } @@ -1382,6 +1391,32 @@ private boolean deleteBackupChain(List chain, long backupScheduleId) { return true; } + private void markRetentionCleanupFailure(List backups, long backupScheduleId, String reason) { + if (CollectionUtils.isEmpty(backups)) { + return; + } + + String failedAt = DateUtil.displayDateInTimezone(DateUtil.GMT_TIMEZONE, new Date()); + String scheduleId = backupScheduleId > 0 ? String.valueOf(backupScheduleId) : ""; + String cleanupFailureReason = StringUtils.abbreviate(StringUtils.defaultString(reason, "unknown"), 1024); + + for (BackupVO backup : backups) { + try { + backupDetailsDao.removeDetail(backup.getId(), RETENTION_CLEANUP_FAILED); + backupDetailsDao.addDetail(backup.getId(), RETENTION_CLEANUP_FAILED, Boolean.TRUE.toString(), false); + backupDetailsDao.removeDetail(backup.getId(), RETENTION_CLEANUP_FAILED_AT); + backupDetailsDao.addDetail(backup.getId(), RETENTION_CLEANUP_FAILED_AT, failedAt, false); + backupDetailsDao.removeDetail(backup.getId(), RETENTION_CLEANUP_SCHEDULE_ID); + backupDetailsDao.addDetail(backup.getId(), RETENTION_CLEANUP_SCHEDULE_ID, scheduleId, false); + backupDetailsDao.removeDetail(backup.getId(), RETENTION_CLEANUP_REASON); + backupDetailsDao.addDetail(backup.getId(), RETENTION_CLEANUP_REASON, cleanupFailureReason, false); + } catch (RuntimeException e) { + logger.warn("Failed to mark retention cleanup failure details for backup [ID: {}, UUID: {}].", + backup.getId(), backup.getUuid(), e); + } + } + } + private List> getBackupChainsForSchedule(List backups) { if (CollectionUtils.isEmpty(backups)) { return new ArrayList<>();