Skip to content

[Europa] 백업 삭제 timeout 적용, 스케줄 백업 생성과 자동 삭제 분리 #939

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Dajeong-Park wants to merge 2 commits into
base: ablestack-europa
Choose a base branch
from
Open

[Europa] 백업 삭제 timeout 적용, 스케줄 백업 생성과 자동 삭제 분리 #939

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
928aa8e
백업 유지 개수 자동 삭제 timeout 처리 개선
Dajeong-Park Sep 7, 2026
51ef9dc
백업 삭제 실패된 백업본 details에 추가
Dajeong-Park Sep 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ...ommvault/src/main/java/org/apache/cloudstack/backup/AblestackCommvaultBackupProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2627,6 +2627,10 @@ private void cleanupBackupPathOnStageHost(String clientName, String path, boolea
throw new CloudRuntimeException(String.format("Unable to find stage host [%s] for backup cleanup", clientName));
}
AblestackDeleteBackupCommand command = new AblestackDeleteBackupCommand(path, null, null, null, forced);
final int commandTimeout = BackupCommandTimeout.value();
if (commandTimeout > 0) {
command.setWait(commandTimeout);
}
command.setBackupProvider("ablestack-commvault");
command.setVmName(vmName);
command.setCheckpointName(checkpointName);
Expand Down
4 changes: 4 additions & 0 deletions .../ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1192,6 +1192,10 @@ public boolean deleteBackup(Backup backup, boolean forced) {

AblestackDeleteBackupCommand command = new AblestackDeleteBackupCommand(backup.getExternalId(), backupRepository.getType(),
backupRepository.getAddress(), backupRepository.getMountOptions(), forced);
final int commandTimeout = BackupCommandTimeout.value();
if (commandTimeout > 0) {
command.setWait(commandTimeout);
}
command.setMountTimeout(NASBackupRestoreMountTimeout.value());
command.setBackupProvider("ablestack-nas");
command.setVmName(vm != null ? vm.getInstanceName() : null);
Expand Down
8 changes: 8 additions & 0 deletions ...tack-netbackup/src/main/java/org/apache/cloudstack/backup/AblestackNetBackupProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,10 @@ private boolean cleanupFailedBackupArtifacts(final Host host, final Backup backu
&& StringUtils.isNotBlank(getBackupDetail(backup, DETAIL_CHECKPOINT_NAME))
&& StringUtils.isNotBlank(getBackupDetail(backup, DETAIL_RBD_DISK_PATHS))) {
final AblestackDeleteBackupCommand command = new AblestackDeleteBackupCommand(backup.getExternalId(), null, null, null, true);
final int commandTimeout = BackupCommandTimeout.value();
if (commandTimeout > 0) {
command.setWait(commandTimeout);
}
command.setBackupProvider(getName());
final VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId());
command.setVmName(vm != null ? vm.getInstanceName() : null);
Expand Down Expand Up @@ -2097,6 +2101,10 @@ private void cleanupExpiredBackupArtifact(final Backup backup, final Set<Long> b
}

final AblestackDeleteBackupCommand command = new AblestackDeleteBackupCommand(backup.getExternalId(), null, null, null, true);
final int commandTimeout = BackupCommandTimeout.value();
if (commandTimeout > 0) {
command.setWait(commandTimeout);
}
command.setBackupProvider(getName());
final VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId());
command.setVmName(vm != null ? vm.getInstanceName() : null);
Expand Down
5 changes: 4 additions & 1 deletion ...com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackDeleteBackupCommandWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public Answer execute(AblestackDeleteBackupCommand command, LibvirtComputingReso
final String cleanupCheckpointNames = command.getCleanupCheckpointNames();
final String diskPaths = command.getDiskPaths();
final boolean forced = command.isForced();
int timeout = command.getWait() > 0 ? command.getWait() * 1000 : libvirtComputingResource.getCmdsTimeout();

List<String[]> commands = new ArrayList<>();
if ("ablestack-commvault".equalsIgnoreCase(backupProvider)) {
Expand Down Expand Up @@ -136,7 +137,9 @@ public Answer execute(AblestackDeleteBackupCommand command, LibvirtComputingReso
commands.add(deleteCommand.toArray(new String[0]));
}

Pair<Integer, String> result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout());
logger.debug("Starting Ablestack backup delete for provider [{}], path [{}] with timeout [{}] ms",
backupProvider, backupPath, timeout);
Pair<Integer, String> result = Script.executePipedCommands(commands, timeout);

logger.debug(String.format("Backup delete result: %s , exit code: %s", result.second(), result.first()));

Expand Down
44 changes: 42 additions & 2 deletions server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,10 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
private static final String FAST_CLONE_FLATTEN_STATUS = "clone.fast.flatten.status";
private static final String FAST_CLONE_FLATTEN_PENDING = "pending";
private static final String FAST_CLONE_FLATTEN_RUNNING = "running";
private static final String RETENTION_CLEANUP_FAILED = "retention.cleanup.failed";
private static final String RETENTION_CLEANUP_FAILED_AT = "retention.cleanup.failed.at";
private static final String RETENTION_CLEANUP_SCHEDULE_ID = "retention.cleanup.schedule.id";
private static final String RETENTION_CLEANUP_REASON = "retention.cleanup.reason";

private static Map<String, BackupProvider> backupProvidersMap = new HashMap<>();
private static final String ABLESTACK_NETBACKUP_PROVIDER_NAME = "ablestack-netbackup";
Expand Down Expand Up @@ -1034,7 +1038,12 @@ public boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllo
}
createCheckedBackup(cmd, owner, isScheduledBackup, backupSize, vm, vmId, backupProvider, backupScheduleId);
if (isScheduledBackup) {
deleteOldestBackupFromScheduleIfRequired(vmId, backupScheduleId);
try {
deleteOldestBackupFromScheduleIfRequired(vmId, backupScheduleId);
} catch (RuntimeException e) {
logger.warn("Failed to apply backup retention cleanup after creating scheduled backup for VM [ID: {}], schedule [ID: {}]. " +
"The backup creation flow will not be failed by this cleanup error.", vmId, backupScheduleId, e);
}
}
logger.info("Completed VM backup request [vmId: {}, vmUuid: {}, vmName: {}, provider: {}, offeringId: {}, scheduleId: {}, elapsedMs: {}]",
vm.getId(), vm.getUuid(), vm.getInstanceName(), offering.getProvider(), offering.getId(), backupScheduleId, System.currentTimeMillis() - backupStartTime);
Expand Down Expand Up @@ -1345,14 +1354,18 @@ private boolean deleteBackupChain(List<BackupVO> chain, long backupScheduleId) {
while (!remainingBackups.isEmpty()) {
List<BackupVO> leafBackups = getLeafBackups(remainingBackups);
if (CollectionUtils.isEmpty(leafBackups)) {
logger.warn("Could not find a deletable leaf while removing an obsolete backup chain for {}.", cleanupTarget);
String reason = "Could not find a deletable leaf while removing an obsolete backup chain";
logger.warn("{} for {}.", reason, cleanupTarget);
markRetentionCleanupFailure(remainingBackups, backupScheduleId, reason);
return false;
}

for (BackupVO backup : leafBackups) {
try {
if (!deleteBackup(backup.getId(), false)) {
String reason = "deleteBackup returned false";
logger.warn("Failed to delete backup [ID: {}, UUID: {}] while deleting a chain for {}.", backup.getId(), backup.getUuid(), cleanupTarget);
markRetentionCleanupFailure(remainingBackups, backupScheduleId, reason);
return false;
}
String eventDescription = backupScheduleId > 0
Expand All @@ -1368,6 +1381,7 @@ private boolean deleteBackupChain(List<BackupVO> chain, long backupScheduleId) {
} catch (Exception e) {
logger.warn("Skipping retention deletion for backup [ID: {}, UUID: {}] on {} because it is not currently safe to remove: {}",
backup.getId(), backup.getUuid(), cleanupTarget, e.getMessage());
markRetentionCleanupFailure(remainingBackups, backupScheduleId, e.getMessage());
return false;
}
}
Expand All @@ -1377,6 +1391,32 @@ private boolean deleteBackupChain(List<BackupVO> chain, long backupScheduleId) {
return true;
}

private void markRetentionCleanupFailure(List<BackupVO> backups, long backupScheduleId, String reason) {
if (CollectionUtils.isEmpty(backups)) {
return;
}

String failedAt = DateUtil.displayDateInTimezone(DateUtil.GMT_TIMEZONE, new Date());
String scheduleId = backupScheduleId > 0 ? String.valueOf(backupScheduleId) : "";
String cleanupFailureReason = StringUtils.abbreviate(StringUtils.defaultString(reason, "unknown"), 1024);

for (BackupVO backup : backups) {
try {
backupDetailsDao.removeDetail(backup.getId(), RETENTION_CLEANUP_FAILED);
backupDetailsDao.addDetail(backup.getId(), RETENTION_CLEANUP_FAILED, Boolean.TRUE.toString(), false);
backupDetailsDao.removeDetail(backup.getId(), RETENTION_CLEANUP_FAILED_AT);
backupDetailsDao.addDetail(backup.getId(), RETENTION_CLEANUP_FAILED_AT, failedAt, false);
backupDetailsDao.removeDetail(backup.getId(), RETENTION_CLEANUP_SCHEDULE_ID);
backupDetailsDao.addDetail(backup.getId(), RETENTION_CLEANUP_SCHEDULE_ID, scheduleId, false);
backupDetailsDao.removeDetail(backup.getId(), RETENTION_CLEANUP_REASON);
backupDetailsDao.addDetail(backup.getId(), RETENTION_CLEANUP_REASON, cleanupFailureReason, false);
} catch (RuntimeException e) {
logger.warn("Failed to mark retention cleanup failure details for backup [ID: {}, UUID: {}].",
backup.getId(), backup.getUuid(), e);
}
}
}

private List<List<BackupVO>> getBackupChainsForSchedule(List<BackupVO> backups) {
if (CollectionUtils.isEmpty(backups)) {
return new ArrayList<>();
Expand Down
Loading