Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(backup): resume unfinished jobs after restart (#5721)

Daniël van Ziel6d96849d2afa
diff --git a/docs/services/backup.md b/docs/services/backup.md
index ca6d9b0106..ea0966809e 100644
--- a/docs/services/backup.md
+++ b/docs/services/backup.md
@@ -110,6 +110,28 @@ FLOCI_SERVICES_BACKUP_JOB_COMPLETION_DELAY_SECONDS=3
Use a shorter delay (e.g. `1`) in test environments to speed up job-completion assertions.
+On startup, persisted `CREATED`, `RUNNING` and `ABORTING` jobs resume in their owning
+account and original region. Existing completion deadlines are retained; older jobs without
+one use their creation time plus the configured delay. A `CREATED` job whose start window
+has elapsed becomes `EXPIRED`. Terminal jobs are not restarted and cannot be stopped.
+
+Completion writes a stable recovery point before marking the job `COMPLETED`. Recovery
+after an interrupted completion reuses that point without changing its original creation
+date, and reconciles the vault count only on this retry path. A new point increments the
+count without scanning the recovery-point store. Completion and recovery-point deletion
+share the vault monitor so retention checks, deletion and count updates stay consistent.
+
+New jobs persist the vault's internal `FlociVaultId` before scheduling.
+Completion requires the original vault ARN and captured id, so deleting and recreating a
+vault with the same name and ARN, even within one second, fails the old job without writing
+a point or notifying the replacement. Legacy jobs without `FlociVaultId` accept only a vault
+no newer than the job; historical dates cannot prove that a same-second vault is the original
+incarnation. Recovery never
+fills a missing job id from the currently resolved vault. The internal identity field stays
+in persisted JSON but is omitted from `DescribeBackupJob` and `ListBackupJobs` responses.
+Successful resumed completions use the existing SNS notification path and event filter;
+publish failures do not undo the job or recovery point.
+
## Supported Resource Types
`GetSupportedResourceTypes` returns the following resource type codes:
diff --git a/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java b/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java
index 41f0be9d78..6788ef373b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java
@@ -416,7 +416,7 @@ public class BackupController {
@GET
@Path("/backup-jobs/{backupJobId}")
public Response describeBackupJob(@PathParam("backupJobId") String jobId) {
- return Response.ok(service.describeBackupJob(jobId)).build();
+ return Response.ok(jobJson(service.describeBackupJob(jobId))).build();
}
@POST
@@ -435,10 +435,16 @@ public class BackupController {
List<BackupJob> jobs = service.listBackupJobs(byVaultName, byState, byResourceArn, byResourceType);
ObjectNode out = objectMapper.createObjectNode();
ArrayNode list = out.putArray("BackupJobs");
- jobs.forEach(list::addPOJO);
+ jobs.forEach(job -> list.add(jobJson(job)));
return Response.ok(out).build();
}
+ private ObjectNode jobJson(BackupJob job) {
+ ObjectNode node = objectMapper.valueToTree(job);
+ node.remove(BackupVault.VAULT_ID_FIELD);
+ return node;
+ }
+
// ── Recovery Point ─────────────────────────────────────────────────────────
@GET
diff --git a/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java b/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java
index daad128344..6edcc9d61b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java
@@ -8,11 +8,21 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
-import io.github.hectorvent.floci.services.backup.model.*;
+import io.github.hectorvent.floci.services.backup.model.BackupJob;
+import io.github.hectorvent.floci.services.backup.model.BackupPlan;
+import io.github.hectorvent.floci.services.backup.model.BackupRule;
+import io.github.hectorvent.floci.services.backup.model.BackupSelection;
+import io.github.hectorvent.floci.services.backup.model.BackupVault;
+import io.github.hectorvent.floci.services.backup.model.BackupVaultNotifications;
+import io.github.hectorvent.floci.services.backup.model.Lifecycle;
+import io.github.hectorvent.floci.services.backup.model.RecoveryPoint;
import io.github.hectorvent.floci.services.sns.SnsService;
import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;
+import io.quarkus.runtime.Startup;
+import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -21,13 +31,19 @@ import org.jboss.logging.Logger;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@ApplicationScoped
+@Startup
public class BackupService {
private static final Logger LOG = Logger.getLogger(BackupService.class);
@@ -40,7 +56,7 @@ public class BackupService {
private final StorageBackend<String, BackupVault> vaultStore;
private final StorageBackend<String, BackupPlan> planStore;
private final StorageBackend<String, BackupSelection> selectionStore;
- private final StorageBackend<String, BackupJob> jobStore;
+ private final AccountAwareStorageBackend<BackupJob> jobStore;
private final StorageBackend<String, RecoveryPoint> recoveryStore;
private final StorageBackend<String, String> accessPolicyStore;
private final StorageBackend<String, BackupVaultNotifications> notificationStore;
@@ -86,6 +102,53 @@ public class BackupService {
scheduler.shutdownNow();
}
+ @PostConstruct
+ void resumeJobs() {
+ for (AccountAwareStorageBackend.AccountEntry<BackupJob> entry : jobStore.scanAllAccountEntries(k -> true)) {
+ BackupJob job = entry.value();
+ if (!List.of("CREATED", "RUNNING", "ABORTING").contains(job.getState())) {
+ continue;
+ }
+ String region = AwsArnUtils.regionOrDefault(job.getBackupVaultArn(), regionResolver.getDefaultRegion());
+ if ("ABORTING".equals(job.getState())) {
+ scheduler.execute(() -> RequestScopes.runAs(entry.accountId(), region, () -> abortJob(entry.key())));
+ } else {
+ scheduleJob(entry.key(), job, entry.accountId(), region);
+ }
+ }
+ }
+
+ private void scheduleJob(String jobId, BackupJob job, String accountId, String region) {
+ long now = Instant.now().getEpochSecond();
+ if ("CREATED".equals(job.getState()) && job.getStartBy() != null && job.getStartBy() < now) {
+ scheduler.execute(() -> RequestScopes.runAs(accountId, region, () -> {
+ synchronized (lockFor(jobLockKey(jobId))) {
+ jobStore.get(jobId).ifPresent(current -> {
+ if ("CREATED".equals(current.getState())) {
+ current.setState("EXPIRED");
+ current.setStatusMessage("The job did not start before its start window expired");
+ current.setCompletionDate(Instant.now().getEpochSecond());
+ jobStore.put(jobId, current);
+ }
+ });
+ }
+ }));
+ return;
+ }
+ long completion = job.getExpectedCompletionDate() == null
+ ? job.getCreationDate() + jobCompletionDelaySeconds : job.getExpectedCompletionDate();
+ scheduler.schedule(() -> RequestScopes.runAs(accountId, region,
+ () -> transitionJob(jobId, job.getBackupVaultName(), region)),
+ Math.max(0, Math.min(job.getCreationDate() + 1, completion) - now), TimeUnit.SECONDS);
+ scheduler.schedule(() -> RequestScopes.runAs(accountId, region,
+ () -> completeJob(jobId, job.getBackupVaultName(), region)),
+ Math.max(0, completion - now), TimeUnit.SECONDS);
+ }
+
+ private String jobLockKey(String jobId) {
+ return "job:" + regionResolver.getAccountId() + ":" + jobId;
+ }
+
// ── Vault ──────────────────────────────────────────────────────────────────
public BackupVault createBackupVault(String vaultName, String encryptionKeyArn,
@@ -653,6 +716,7 @@ public class BackupService {
job.setBackupJobId(jobId);
job.setBackupVaultName(vaultName);
job.setBackupVaultArn(vault.getBackupVaultArn());
+ job.setVaultId(vault.getVaultId());
job.setResourceArn(resourceArn);
job.setResourceType(inferResourceType(resourceArn));
job.setIamRoleArn(iamRoleArn);
@@ -664,16 +728,7 @@ public class BackupService {
String accountId = regionResolver.getAccountId();
job.setAccountId(accountId);
jobStore.put(jobId, job);
- // Pinned now, not looked up by name at completion: if this vault is deleted and the name
- // reused while the job runs, the job must not notify the replacement vault's topic.
- String notificationKey = subResourceKey(region, vault);
-
- // The scheduler thread has no request context; run in the caller's account
- // so the account-aware stores and buildArn resolve the right partition.
- scheduler.schedule(() -> RequestScopes.runAs(accountId,
- () -> transitionJob(jobId, vaultName, region)), 1, TimeUnit.SECONDS);
- scheduler.schedule(() -> RequestScopes.runAs(accountId,
- () -> completeJob(jobId, vaultName, region, notificationKey)), jobCompletionDelaySeconds, TimeUnit.SECONDS);
+ scheduleJob(jobId, job, accountId, region);
return job;
}
@@ -684,17 +739,20 @@ public class BackupService {
}
public void stopBackupJob(String jobId) {
- BackupJob job = describeBackupJob(jobId);
- String state = job.getState();
- if ("COMPLETED".equals(state) || "ABORTED".equals(state) || "FAILED".equals(state)) {
- throw new AwsException("InvalidRequestException",
- "Job cannot be stopped in state: " + state, 400);
+ synchronized (lockFor(jobLockKey(jobId))) {
+ BackupJob job = describeBackupJob(jobId);
+ String state = job.getState();
+ if (List.of("COMPLETED", "ABORTED", "FAILED", "EXPIRED", "PARTIAL").contains(state)) {
+ throw new AwsException("InvalidRequestException",
+ "Job cannot be stopped in state: " + state, 400);
+ }
+ job.setState("ABORTING");
+ job.setStatusMessage("Job stop requested");
+ jobStore.put(jobId, job);
+ String accountId = regionResolver.getAccountId();
+ String region = AwsArnUtils.regionOrDefault(job.getBackupVaultArn(), regionResolver.getDefaultRegion());
+ scheduler.schedule(() -> RequestScopes.runAs(accountId, region, () -> abortJob(jobId)), 1, TimeUnit.SECONDS);
}
- job.setState("ABORTING");
- job.setStatusMessage("Job stop requested");
- jobStore.put(jobId, job);
- String accountId = regionResolver.getAccountId();
- scheduler.schedule(() -> RequestScopes.runAs(accountId, () -> abortJob(jobId)), 1, TimeUnit.SECONDS);
}
public List<BackupJob> listBackupJobs(String byVaultName, String byState,
@@ -743,20 +801,22 @@ public class BackupService {
* passes would be describing AWS behaviour that needs a specific permission to reproduce.
*/
public void deleteRecoveryPoint(String vaultName, String recoveryPointArn, String region) {
- BackupVault vault = describeBackupVault(vaultName, region);
- RecoveryPoint rp = describeRecoveryPoint(vaultName, recoveryPointArn, region);
- if (vault.isLocked() && vault.getMinRetentionDays() != null) {
- long ageDays = ChronoUnit.DAYS.between(
- Instant.ofEpochSecond(rp.getCreationDate()), Instant.now());
- if (ageDays < vault.getMinRetentionDays()) {
- throw new AwsException("InvalidRequestException",
- "Recovery point is protected by a vault lock until its minimum retention of "
- + vault.getMinRetentionDays() + " day(s) has elapsed: " + recoveryPointArn,
- 400);
+ synchronized (lockFor(vaultKey(region, vaultName))) {
+ BackupVault vault = describeBackupVault(vaultName, region);
+ RecoveryPoint rp = describeRecoveryPoint(vaultName, recoveryPointArn, region);
+ if (vault.isLocked() && vault.getMinRetentionDays() != null) {
+ long ageDays = ChronoUnit.DAYS.between(
+ Instant.ofEpochSecond(rp.getCreationDate()), Instant.now());
+ if (ageDays < vault.getMinRetentionDays()) {
+ throw new AwsException("InvalidRequestException",
+ "Recovery point is protected by a vault lock until its minimum retention of "
+ + vault.getMinRetentionDays() + " day(s) has elapsed: " + recoveryPointArn,
+ 400);
+ }
}
+ recoveryStore.delete(recoveryPointArn);
+ decrementVaultCount(vaultName, region);
}
- recoveryStore.delete(recoveryPointArn);
- decrementVaultCount(vaultName, region);
}
// ── Tags ───────────────────────────────────────────────────────────────────
@@ -782,24 +842,28 @@ public class BackupService {
// ── Private helpers ────────────────────────────────────────────────────────
private void transitionJob(String jobId, String vaultName, String region) {
- jobStore.get(jobId).ifPresent(job -> {
- if ("CREATED".equals(job.getState())) {
- job.setState("RUNNING");
- job.setPercentDone("50.0");
- jobStore.put(jobId, job);
- }
- });
+ synchronized (lockFor(jobLockKey(jobId))) {
+ jobStore.get(jobId).ifPresent(job -> {
+ if ("CREATED".equals(job.getState())) {
+ job.setState("RUNNING");
+ job.setPercentDone("50.0");
+ jobStore.put(jobId, job);
+ }
+ });
+ }
}
- private void completeJob(String jobId, String vaultName, String region, String notificationKey) {
+ private void completeJob(String jobId, String vaultName, String region) {
// Under the vault-name monitor, so the vault cannot be deleted or recreated between the
// incarnation check and the recovery point landing in it.
- synchronized (lockFor(vaultKey(region, vaultName))) {
- completeJobLocked(jobId, vaultName, region, notificationKey);
+ synchronized (lockFor(jobLockKey(jobId))) {
+ synchronized (lockFor(vaultKey(region, vaultName))) {
+ completeJobLocked(jobId, vaultName, region);
+ }
}
}
- private void completeJobLocked(String jobId, String vaultName, String region, String notificationKey) {
+ private void completeJobLocked(String jobId, String vaultName, String region) {
jobStore.get(jobId).ifPresent(job -> {
if ("RUNNING".equals(job.getState())) {
long now = Instant.now().getEpochSecond();
@@ -807,9 +871,10 @@ public class BackupService {
// vault holds the name now. A vault deleted mid-job (allowed: it held no recovery
// point yet) and recreated unlocked would otherwise receive a recovery point its
// creator's lock was meant to protect, deletable inside that lock's retention.
- boolean sameVault = vaultStore.get(vaultKey(region, vaultName))
- .map(v -> subResourceKey(region, v).equals(notificationKey))
- .orElse(false);
+ Optional<BackupVault> vault = vaultStore.get(vaultKey(region, vaultName));
+ boolean sameVault = vault.filter(v -> v.getBackupVaultArn().equals(job.getBackupVaultArn())
+ && (job.getVaultId() != null ? job.getVaultId().equals(v.getVaultId())
+ : v.getCreationDate() <= job.getCreationDate())).isPresent();
if (!sameVault) {
job.setState("FAILED");
job.setStatusMessage("Backup vault was deleted before the job completed: " + vaultName);
@@ -817,9 +882,31 @@ public class BackupService {
jobStore.put(jobId, job);
return;
}
- String rpArn = regionResolver.buildArn("backup", region,
- "recovery-point:" + UUID.randomUUID());
-
+ BackupVault currentVault = vault.orElseThrow();
+ String notificationKey = subResourceKey(region, currentVault);
+ String rpArn = job.getRecoveryPointArn() != null ? job.getRecoveryPointArn()
+ : regionResolver.buildArn("backup", region, "recovery-point:" + jobId);
+ if (recoveryStore.get(rpArn).isEmpty()) {
+ RecoveryPoint rp = new RecoveryPoint();
+ rp.setRecoveryPointArn(rpArn);
+ rp.setBackupVaultName(vaultName);
+ rp.setBackupVaultArn(job.getBackupVaultArn());
+ rp.setResourceArn(job.getResourceArn());
+ rp.setResourceType(job.getResourceType());
+ rp.setIamRoleArn(job.getIamRoleArn());
+ rp.setStatus("COMPLETED");
+ rp.setCreationDate(job.getCreationDate());
+ rp.setCompletionDate(now);
+ rp.setBackupSizeInBytes(0L);
+ rp.setStorageClass("WARM");
+ rp.setEncrypted(false);
+ recoveryStore.put(rpArn, rp);
+ currentVault.setNumberOfRecoveryPoints(currentVault.getNumberOfRecoveryPoints() + 1);
+ } else {
+ currentVault.setNumberOfRecoveryPoints((int) recoveryStore.scan(k -> true).stream()
+ .filter(point -> currentVault.getBackupVaultArn().equals(point.getBackupVaultArn())).count());
+ }
+ vaultStore.put(vaultKey(region, vaultName), currentVault);
job.setState("COMPLETED");
job.setPercentDone("100.0");
job.setCompletionDate(now);
@@ -827,23 +914,6 @@ public class BackupService {
job.setBackupSizeInBytes(0L);
job.setBytesTransferred(0L);
jobStore.put(jobId, job);
-
- RecoveryPoint rp = new RecoveryPoint();
- rp.setRecoveryPointArn(rpArn);
- rp.setBackupVaultName(vaultName);
- rp.setBackupVaultArn(job.getBackupVaultArn());
- rp.setResourceArn(job.getResourceArn());
- rp.setResourceType(job.getResourceType());
- rp.setIamRoleArn(job.getIamRoleArn());
- rp.setStatus("COMPLETED");
- rp.setCreationDate(job.getCreationDate());
- rp.setCompletionDate(now);
- rp.setBackupSizeInBytes(0L);
- rp.setStorageClass("WARM");
- rp.setEncrypted(false);
- recoveryStore.put(rpArn, rp);
-
- incrementVaultCount(vaultName, region);
LOG.infov("Backup job {0} completed, recovery point: {1}", jobId, rpArn);
notifyJobCompleted(job, vaultName, region, notificationKey);
}
@@ -885,20 +955,15 @@ public class BackupService {
}
private void abortJob(String jobId) {
- jobStore.get(jobId).ifPresent(job -> {
- if ("ABORTING".equals(job.getState())) {
- job.setState("ABORTED");
- job.setCompletionDate(Instant.now().getEpochSecond());
- jobStore.put(jobId, job);
- }
- });
- }
-
- private void incrementVaultCount(String vaultName, String region) {
- vaultStore.get(vaultKey(region, vaultName)).ifPresent(vault -> {
- vault.setNumberOfRecoveryPoints(vault.getNumberOfRecoveryPoints() + 1);
- vaultStore.put(vaultKey(region, vaultName), vault);
- });
+ synchronized (lockFor(jobLockKey(jobId))) {
+ jobStore.get(jobId).ifPresent(job -> {
+ if ("ABORTING".equals(job.getState())) {
+ job.setState("ABORTED");
+ job.setCompletionDate(Instant.now().getEpochSecond());
+ jobStore.put(jobId, job);
+ }
+ });
+ }
}
private void decrementVaultCount(String vaultName, String region) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupJob.java b/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupJob.java
index d5dd8d7b50..341ddfee9a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupJob.java
+++ b/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupJob.java
@@ -19,6 +19,9 @@ public class BackupJob {
@JsonProperty("BackupVaultArn")
private String backupVaultArn;
+ @JsonProperty("FlociVaultId")
+ private String vaultId;
+
@JsonProperty("RecoveryPointArn")
private String recoveryPointArn;
@@ -72,6 +75,9 @@ public class BackupJob {
public String getBackupVaultArn() { return backupVaultArn; }
public void setBackupVaultArn(String backupVaultArn) { this.backupVaultArn = backupVaultArn; }
+ public String getVaultId() { return vaultId; }
+ public void setVaultId(String vaultId) { this.vaultId = vaultId; }
+
public String getRecoveryPointArn() { return recoveryPointArn; }
public void setRecoveryPointArn(String recoveryPointArn) { this.recoveryPointArn = recoveryPointArn; }
diff --git a/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java
index ad7b73cbb2..8a93bebdae 100644
--- a/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java
@@ -1,8 +1,12 @@
package io.github.hectorvent.floci.services.backup;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.services.backup.model.BackupJob;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.path.json.JsonPath;
import io.restassured.specification.RequestSpecification;
+import jakarta.inject.Inject;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
@@ -31,6 +35,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class BackupIntegrationTest {
+ @Inject BackupService backupService;
+ @Inject ObjectMapper mapper;
+
private static final String AUTH = "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/backup/aws4_request";
private static final String VAULT_NAME = "test-vault";
private static final String IAM_ROLE = "arn:aws:iam::000000000000:role/backup-role";
@@ -1941,4 +1948,45 @@ class BackupIntegrationTest {
.body("NumberOfRecoveryPoints", equalTo(0));
}
+ @Test
+ @Order(201)
+ void describeAndListJobsHideCapturedIdentityWithoutRemovingStoredOrPublicFields() {
+ String vault = "job-api-identity-vault";
+ given().header("Authorization", AUTH).contentType("application/json").body("{}")
+ .when().put("/backup-vaults/" + vault).then().statusCode(200);
+ String jid = given().header("Authorization", AUTH).contentType("application/json")
+ .body("""
+ {"BackupVaultName":"%s","ResourceArn":"%s","IamRoleArn":"%s"}
+ """.formatted(vault, RESOURCE_ARN, IAM_ROLE))
+ .when().put("/backup-jobs").then().statusCode(200).extract().path("BackupJobId");
+ BackupJob stored = backupService.describeBackupJob(jid);
+ String capturedId = backupService.describeBackupVault(vault, "us-east-1").getVaultId();
+ assertEquals(capturedId, stored.getVaultId());
+ JsonNode before = mapper.valueToTree(stored);
+ assertEquals(capturedId, before.path("FlociVaultId").asText());
+ given().header("Authorization", AUTH)
+ .when().get("/backup-jobs/" + jid).then().statusCode(200)
+ .body("$", not(hasKey("FlociVaultId")))
+ .body("BackupJobId", equalTo(jid))
+ .body("BackupVaultName", equalTo(vault))
+ .body("BackupVaultArn", equalTo(stored.getBackupVaultArn()))
+ .body("ResourceArn", equalTo(RESOURCE_ARN))
+ .body("CreationDate", notNullValue())
+ .body("State", notNullValue());
+ given().header("Authorization", AUTH).queryParam("byBackupVaultName", vault)
+ .when().get("/backup-jobs/").then().statusCode(200)
+ .body("BackupJobs", hasSize(1))
+ .body("BackupJobs[0]", not(hasKey("FlociVaultId")))
+ .body("BackupJobs[0].BackupJobId", equalTo(jid))
+ .body("BackupJobs[0].BackupVaultName", equalTo(vault))
+ .body("BackupJobs[0].ResourceArn", equalTo(RESOURCE_ARN))
+ .body("BackupJobs[0].CreationDate", notNullValue())
+ .body("BackupJobs[0].State", notNullValue());
+ JsonNode after = mapper.valueToTree(backupService.describeBackupJob(jid));
+ assertEquals(before.path("FlociVaultId"), after.path("FlociVaultId"));
+ awaitJobCompleted(jid);
+ String point = backupService.describeBackupJob(jid).getRecoveryPointArn();
+ backupService.deleteRecoveryPoint(vault, point, "us-east-1");
+ backupService.deleteBackupVault(vault, "us-east-1");
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryIntegrationTest.java
new file mode 100644
index 0000000000..bb02e0ac86
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryIntegrationTest.java
@@ -0,0 +1,79 @@
+package io.github.hectorvent.floci.services.backup;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.backup.model.BackupJob;
+import io.github.hectorvent.floci.services.sns.SnsService;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.*;
+
+@QuarkusTest
+class BackupJobRecoveryIntegrationTest {
+ @Inject StorageFactory storage;
+ @Inject EmulatorConfig config;
+ @Inject RegionResolver resolver;
+ @Inject ObjectMapper mapper;
+ @Inject SnsService sns;
+
+ @Test
+ void restartRestoresJobsInEveryAccountAndTheirOriginalRegion() {
+ String vaultName = "restart-" + UUID.randomUUID();
+ List<String> accounts = List.of("811111111111", "822222222222");
+ BackupService first = new BackupService(storage, config, resolver, mapper, sns);
+ AccountAwareStorageBackend<BackupJob> jobs = storage.create("backup", "backup-jobs.json",
+ new TypeReference<Map<String, BackupJob>>() {});
+ try {
+ for (String account : accounts) {
+ RequestScopes.runAs(account, "eu-west-1", () -> {
+ first.createBackupVault(vaultName, null, null, Map.of(), "eu-west-1");
+ BackupJob job = first.startBackupJob(vaultName, "arn:aws:s3:::bucket", "role", null, "eu-west-1");
+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);
+ jobs.put(job.getBackupJobId(), job);
+ });
+ }
+ } finally {
+ first.shutdown();
+ }
+ BackupService restarted = new BackupService(storage, config, resolver, mapper, sns);
+ try {
+ restarted.resumeJobs();
+ for (String account : accounts) {
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> RequestScopes.runAs(account, () -> {
+ assertEquals("COMPLETED", restarted.listBackupJobs(vaultName, null, null, null).getFirst().getState());
+ assertEquals(1, restarted.listRecoveryPointsByBackupVault(vaultName, "eu-west-1").size());
+ String arn = restarted.listRecoveryPointsByBackupVault(vaultName, "eu-west-1")
+ .getFirst().getRecoveryPointArn();
+ assertTrue(arn.contains(":eu-west-1:" + account + ":"));
+ assertEquals(1, restarted.describeBackupVault(vaultName, "eu-west-1").getNumberOfRecoveryPoints());
+ assertEquals(restarted.describeBackupVault(vaultName, "eu-west-1").getVaultId(),
+ restarted.listBackupJobs(vaultName, null, null, null).getFirst().getVaultId());
+ }));
+ }
+ } finally {
+ restarted.shutdown();
+ for (String account : accounts) {
+ RequestScopes.runAs(account, () -> {
+ restarted.listRecoveryPointsByBackupVault(vaultName, "eu-west-1").forEach(point ->
+ restarted.deleteRecoveryPoint(vaultName, point.getRecoveryPointArn(), "eu-west-1"));
+ restarted.listBackupJobs(vaultName, null, null, null).forEach(job -> jobs.delete(job.getBackupJobId()));
+ restarted.deleteBackupVault(vaultName, "eu-west-1");
+ });
+ }
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryTest.java b/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryTest.java
new file mode 100644
index 0000000000..237eb1021e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryTest.java
@@ -0,0 +1,655 @@
+package io.github.hectorvent.floci.services.backup;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.ServiceConfigAccess;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.backup.model.BackupJob;
+import io.github.hectorvent.floci.services.backup.model.BackupVault;
+import io.github.hectorvent.floci.services.backup.model.RecoveryPoint;
+import io.github.hectorvent.floci.services.sns.SnsService;
+import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+class BackupJobRecoveryTest {
+ private static final String ACCOUNT = "123456789012";
+ private static final String REGION = "eu-west-1";
+ private static final String VAULT = "review";
+ private static final String TOPIC = "arn:aws:sns:eu-west-1:" + ACCOUNT + ":backup";
+
+ @Test
+ void restartResumesEachNonTerminalStateAndDoesNotDuplicateRecoveryPoints() {
+ try (Fixture fixture = new Fixture()) {
+ fixture.vault().setCreationDate(Instant.now().getEpochSecond() - 60);
+ BackupJob original = fixture.service.startBackupJob(VAULT, "arn:aws:s3:::bucket", "role", null, REGION);
+ fixture.service.shutdown();
+ long now = Instant.now().getEpochSecond();
+ original.setExpectedCompletionDate(now - 1);
+ fixture.jobs.put(original.getBackupJobId(), original);
+ for (String state : List.of("RUNNING", "ABORTING", "COMPLETED", "FAILED", "ABORTED")) {
+ BackupJob job = fixture.legacy(state, now - 10);
+ job.setBackupJobId(state);
+ fixture.jobs.put(state, job);
+ }
+ fixture.jobs.delete("legacy");
+ BackupJob expired = fixture.legacy("CREATED", now - 10);
+ expired.setBackupJobId("expired");
+ expired.setStartBy(now - 1);
+ fixture.jobs.put("expired", expired);
+ BackupJob missing = fixture.legacy("RUNNING", now - 10);
+ missing.setBackupJobId("missing-vault");
+ missing.setBackupVaultName("deleted");
+ fixture.jobs.put("missing-vault", missing);
+ BackupJob future = fixture.legacy("RUNNING", now - 10);
+ future.setBackupJobId("future");
+ future.setExpectedCompletionDate(now + 3600);
+ fixture.jobs.put("future", future);
+ fixture.jobs.delete("legacy");
+ RecoveryPoint existing = new RecoveryPoint();
+ existing.setBackupVaultName(VAULT);
+ existing.setBackupVaultArn(fixture.vault().getBackupVaultArn());
+ existing.setRecoveryPointArn("arn:aws:backup:" + REGION + ":" + ACCOUNT + ":recovery-point:RUNNING");
+ existing.setCreationDate(now - 20);
+ fixture.points.put(existing.getRecoveryPointArn(), existing);
+ fixture.service = fixture.newService();
+ fixture.service.resumeJobs();
+ assertEquals("RUNNING", fixture.service.describeBackupJob("future").getState());
+ assertEquals(now + 3600, fixture.service.describeBackupJob("future").getExpectedCompletionDate());
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> {
+ assertEquals("COMPLETED", fixture.service.describeBackupJob(original.getBackupJobId()).getState());
+ assertEquals("COMPLETED", fixture.service.describeBackupJob("RUNNING").getState());
+ assertEquals("ABORTED", fixture.service.describeBackupJob("ABORTING").getState());
+ assertEquals("EXPIRED", fixture.service.describeBackupJob("expired").getState());
+ assertEquals("FAILED", fixture.service.describeBackupJob("missing-vault").getState());
+ });
+ fixture.service.resumeJobs();
+ assertEquals(2, fixture.service.listRecoveryPointsByBackupVault(VAULT, REGION).size());
+ assertEquals(2, fixture.vault().getNumberOfRecoveryPoints());
+ assertEquals(now - 20, fixture.points.get(existing.getRecoveryPointArn()).orElseThrow().getCreationDate());
+ assertTrue(fixture.service.listRecoveryPointsByBackupVault(VAULT, REGION).stream()
+ .allMatch(point -> point.getRecoveryPointArn().contains(":" + REGION + ":" + ACCOUNT + ":")));
+ for (String state : List.of("COMPLETED", "FAILED", "ABORTED")) {
+ assertEquals(state, fixture.service.describeBackupJob(state).getState());
+ assertNull(fixture.service.describeBackupJob(state).getRecoveryPointArn());
+ }
+ }
+ }
+
+ @Test
+ void deletionWaitsForCompletionMonitorAndLeavesCountConsistent() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ BackupJob first = fixture.startRunning();
+ fixture.complete(first);
+ BackupJob second = fixture.startRunning();
+ Object monitor = fixture.vaultMonitor();
+ CountDownLatch entered = new CountDownLatch(1);
+ FutureTask<Void> deletion = new FutureTask<>(() -> {
+ entered.countDown();
+ fixture.service.deleteRecoveryPoint(VAULT, first.getRecoveryPointArn(), REGION);
+ return null;
+ });
+ Thread worker = new Thread(deletion, "backup-review-delete");
+ boolean waited;
+ try {
+ synchronized (monitor) {
+ worker.start();
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ await().atMost(Duration.ofSeconds(5)).until(() ->
+ worker.getState() == Thread.State.BLOCKED || deletion.isDone());
+ waited = worker.getState() == Thread.State.BLOCKED && !deletion.isDone();
+ fixture.complete(second);
+ }
+ deletion.get(5, TimeUnit.SECONDS);
+ assertAll(
+ () -> assertTrue(waited, "Point deletion must wait on completion's vault monitor"),
+ () -> assertEquals(1, fixture.service.listRecoveryPointsByBackupVault(VAULT, REGION).size()),
+ () -> assertEquals(1, fixture.vault().getNumberOfRecoveryPoints()),
+ () -> assertThrows(AwsException.class,
+ () -> fixture.service.deleteBackupVault(VAULT, REGION)));
+ fixture.service.deleteRecoveryPoint(VAULT, second.getRecoveryPointArn(), REGION);
+ assertTrue(fixture.service.listRecoveryPointsByBackupVault(VAULT, REGION).isEmpty());
+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());
+ fixture.service.deleteBackupVault(VAULT, REGION);
+ assertThrows(AwsException.class, fixture::vault);
+ } finally {
+ worker.join(5000);
+ assertFalse(worker.isAlive(), "Deletion worker must be cleaned up");
+ }
+ }
+ }
+
+ @Test
+ void retentionReadPointDeleteAndCountReadWriteAllHoldCompletionMonitor() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ BackupJob job = fixture.startRunning();
+ fixture.complete(job);
+ Object monitor = fixture.vaultMonitor();
+ RecoveryPoint point = fixture.points.get(job.getRecoveryPointArn()).orElseThrow();
+ fixture.service.putBackupVaultLockConfiguration(VAULT, REGION, 1L, null, null);
+ doAnswer(call -> {
+ assertTrue(Thread.holdsLock(monitor), "Retention and count reads must hold the vault monitor");
+ return call.callRealMethod();
+ }).when(fixture.vaults).get(REGION + ":" + VAULT);
+ doAnswer(call -> {
+ assertTrue(Thread.holdsLock(monitor), "Point lookup for retention must hold the vault monitor");
+ return call.callRealMethod();
+ }).when(fixture.points).get(job.getRecoveryPointArn());
+ doAnswer(call -> {
+ assertTrue(Thread.holdsLock(monitor), "Point deletion must hold the vault monitor");
+ return call.callRealMethod();
+ }).when(fixture.points).delete(job.getRecoveryPointArn());
+ doAnswer(call -> {
+ assertTrue(Thread.holdsLock(monitor), "Count update must hold the vault monitor");
+ return call.callRealMethod();
+ }).when(fixture.vaults).put(eq(REGION + ":" + VAULT), any());
+ AwsException protectedPoint = assertThrows(AwsException.class,
+ () -> fixture.service.deleteRecoveryPoint(VAULT, job.getRecoveryPointArn(), REGION));
+ assertEquals("InvalidRequestException", protectedPoint.getErrorCode());
+ verify(fixture.points, never()).delete(anyString());
+ point.setCreationDate(Instant.now().minus(Duration.ofDays(2)).getEpochSecond());
+ fixture.service.deleteRecoveryPoint(VAULT, job.getRecoveryPointArn(), REGION);
+ verify(fixture.points).delete(job.getRecoveryPointArn());
+ doCallRealMethod().when(fixture.vaults).get(anyString());
+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());
+ fixture.service.deleteBackupVault(VAULT, REGION);
+ }
+ }
+
+ @Test
+ void brandNewPointIncrementsCountWithoutScanningRecoveryStore() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ doAnswer(call -> {
+ BackupJob written = call.getArgument(1);
+ if ("COMPLETED".equals(written.getState())) {
+ assertTrue(fixture.points.get(written.getRecoveryPointArn()).isPresent(),
+ "The point must be persisted before the terminal job write");
+ assertTrue(Thread.holdsLock(fixture.vaultMonitor()));
+ }
+ return call.callRealMethod();
+ }).when(fixture.jobs).put(anyString(), any());
+ BackupJob first = fixture.startRunning();
+ fixture.complete(first);
+ BackupJob second = fixture.startRunning();
+ fixture.complete(second);
+ verify(fixture.points, never()).scan(any());
+ assertEquals(2, fixture.vault().getNumberOfRecoveryPoints());
+ assertEquals(2, fixture.points.keys().size());
+ fixture.complete(first);
+ assertEquals(2, fixture.vault().getNumberOfRecoveryPoints());
+ verify(fixture.points, never()).scan(any());
+ }
+ }
+
+ @Test
+ void stablePointRetryRecountsAndRestartDoesNotDuplicateIt() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ BackupJob job = fixture.startRunning();
+ fixture.complete(job);
+ String pointArn = job.getRecoveryPointArn();
+ RecoveryPoint original = fixture.points.get(pointArn).orElseThrow();
+ original.setCreationDate(42L);
+ fixture.vault().setNumberOfRecoveryPoints(9);
+ job.setState("RUNNING");
+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);
+ fixture.jobs.put(job.getBackupJobId(), job);
+ fixture.service.shutdown();
+ clearInvocations(fixture.points);
+ fixture.service = fixture.newService();
+ fixture.service.resumeJobs();
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ assertEquals("COMPLETED", fixture.service.describeBackupJob(job.getBackupJobId()).getState()));
+ verify(fixture.points, atLeastOnce()).scan(any());
+ assertEquals(1, fixture.points.keys().size());
+ assertEquals(1, fixture.vault().getNumberOfRecoveryPoints());
+ assertEquals(42L, fixture.points.get(pointArn).orElseThrow().getCreationDate());
+ assertEquals(pointArn, fixture.service.describeBackupJob(job.getBackupJobId()).getRecoveryPointArn());
+ fixture.service.resumeJobs();
+ fixture.complete(job);
+ assertEquals(1, fixture.points.keys().size());
+ assertEquals(1, fixture.vault().getNumberOfRecoveryPoints());
+ }
+ }
+
+ @Test
+ void newJobPersistsVaultIncarnationBeforeScheduling() {
+ try (Fixture fixture = new Fixture()) {
+ fixture.vault().setCreationDate(100L);
+ doAnswer(call -> {
+ BackupJob written = call.getArgument(1);
+ assertEquals(fixture.vault().getVaultId(), written.getVaultId());
+ return call.callRealMethod();
+ }).when(fixture.jobs).put(anyString(), any());
+ BackupJob job = fixture.startRunning();
+ JsonNode stored = fixture.mapper.valueToTree(fixture.jobs.get(job.getBackupJobId()).orElseThrow());
+ assertEquals(fixture.vault().getVaultId(), stored.path("FlociVaultId").asText(),
+ "New jobs must persist the captured incarnation");
+ }
+ }
+
+ @Test
+ void capturedIncarnationRejectsNewerSameArnReplacement() throws Exception {
+ assertReplacementRejected(200L);
+ }
+
+ @Test
+ void capturedIncarnationRequiresEqualityEvenForOlderSameArnReplacement() throws Exception {
+ assertReplacementRejected(99L);
+ }
+
+ private void assertReplacementRejected(long replacementDate) throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ fixture.vault().setCreationDate(100L);
+ BackupJob job = fixture.startRunning();
+ String originalArn = fixture.vault().getBackupVaultArn();
+ fixture.service.deleteBackupVault(VAULT, REGION);
+ BackupVault replacement = fixture.service.createBackupVault(VAULT, null, null, Map.of(), REGION);
+ replacement.setCreationDate(replacementDate);
+ assertEquals(originalArn, replacement.getBackupVaultArn());
+ assertNotEquals(job.getVaultId(), replacement.getVaultId());
+ fixture.complete(job);
+ assertEquals("FAILED", fixture.service.describeBackupJob(job.getBackupJobId()).getState());
+ assertTrue(fixture.points.keys().isEmpty());
+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());
+ fixture.service.deleteBackupVault(VAULT, REGION);
+ }
+ }
+
+ @Test
+ void legacyAbsentIdentityAllowsOlderAndSameSecondVaults() throws Exception {
+ for (long vaultDate : new long[]{99L, 100L}) {
+ try (Fixture fixture = new Fixture()) {
+ fixture.vault().setCreationDate(vaultDate);
+ BackupJob legacy = fixture.legacy("RUNNING", 100L);
+ fixture.complete(legacy);
+ assertEquals("COMPLETED", fixture.service.describeBackupJob(legacy.getBackupJobId()).getState());
+ assertEquals(1, fixture.vault().getNumberOfRecoveryPoints());
+ assertEquals("arn:aws:backup:" + REGION + ":" + ACCOUNT + ":recovery-point:legacy",
+ legacy.getRecoveryPointArn());
+ assertEquals(ACCOUNT, legacy.getAccountId());
+ assertNull(legacy.getVaultId(), "Recovery must not invent a missing captured id");
+ }
+ }
+ }
+
+ @Test
+ void legacyAbsentIdentityRejectsVaultNewerThanJob() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ fixture.vault().setCreationDate(101L);
+ BackupJob legacy = fixture.legacy("RUNNING", 100L);
+ fixture.complete(legacy);
+ assertEquals("FAILED", fixture.service.describeBackupJob(legacy.getBackupJobId()).getState());
+ assertTrue(fixture.points.keys().isEmpty());
+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());
+ }
+ }
+
+ @Test
+ void restartKeepsLegacyStartAndCompletionDeadlinesInOriginalRegion() {
+ try (Fixture fixture = new Fixture()) {
+ long now = Instant.now().getEpochSecond();
+ fixture.vault().setCreationDate(now - 20);
+ BackupJob expired = fixture.legacy("CREATED", now - 10);
+ expired.setStartBy(now - 1);
+ BackupJob future = fixture.legacy("RUNNING", now - 10);
+ future.setBackupJobId("future");
+ future.setExpectedCompletionDate(now + 3600);
+ fixture.jobs.put("future", future);
+ fixture.jobs.put("legacy", expired);
+ fixture.service.resumeJobs();
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ assertEquals("EXPIRED", fixture.service.describeBackupJob("legacy").getState()));
+ assertEquals("RUNNING", fixture.service.describeBackupJob("future").getState());
+ assertEquals(now + 3600, fixture.service.describeBackupJob("future").getExpectedCompletionDate());
+ assertTrue(fixture.points.keys().isEmpty());
+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());
+ }
+ }
+
+ @Test
+ void sameSecondSameNameSameArnReplacementRejectsCapturedIdWithoutPointsOrNotifications() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ BackupJob job = fixture.startRunning();
+ BackupVault original = fixture.vault();
+ fixture.service.deleteBackupVault(VAULT, REGION);
+ BackupVault replacement = fixture.service.createBackupVault(VAULT, null, null, Map.of(), REGION);
+ replacement.setCreationDate(original.getCreationDate());
+ fixture.service.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_COMPLETED"));
+ assertEquals(original.getBackupVaultName(), replacement.getBackupVaultName());
+ assertEquals(original.getBackupVaultArn(), replacement.getBackupVaultArn());
+ assertEquals(original.getCreationDate(), replacement.getCreationDate());
+ assertNotEquals(original.getVaultId(), replacement.getVaultId());
+ fixture.complete(job);
+ assertEquals("FAILED", fixture.service.describeBackupJob(job.getBackupJobId()).getState());
+ assertTrue(fixture.points.keys().isEmpty());
+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());
+ verifyNoInteractions(fixture.sns);
+ }
+ }
+
+ @Test
+ void jsonPersistenceRoundtripAndServiceRecreationAcceptOriginalAndRejectReplacement() throws Exception {
+ for (boolean replace : List.of(false, true)) {
+ try (Fixture fixture = new Fixture()) {
+ BackupJob job = fixture.startRunning();
+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);
+ BackupVault original = fixture.vault();
+ String capturedId = original.getVaultId();
+ String jobJson = fixture.mapper.writeValueAsString(job);
+ String vaultJson = fixture.mapper.writeValueAsString(original);
+ fixture.service.shutdown();
+ fixture.jobs.put(job.getBackupJobId(), fixture.mapper.readValue(jobJson, BackupJob.class));
+ fixture.vaults.put(REGION + ":" + VAULT, fixture.mapper.readValue(vaultJson, BackupVault.class));
+ fixture.service = fixture.newService();
+ assertEquals(capturedId, fixture.service.describeBackupJob(job.getBackupJobId()).getVaultId());
+ assertEquals(capturedId, fixture.vault().getVaultId());
+ if (replace) {
+ fixture.service.deleteBackupVault(VAULT, REGION);
+ fixture.service.createBackupVault(VAULT, null, null, Map.of(), REGION)
+ .setCreationDate(original.getCreationDate());
+ }
+ fixture.service.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_COMPLETED"));
+ fixture.service.resumeJobs();
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertEquals(
+ replace ? "FAILED" : "COMPLETED",
+ fixture.service.describeBackupJob(job.getBackupJobId()).getState()));
+ assertEquals(capturedId, fixture.service.describeBackupJob(job.getBackupJobId()).getVaultId());
+ assertEquals(replace ? 0 : 1, fixture.points.keys().size());
+ assertEquals(replace ? 0 : 1, fixture.vault().getNumberOfRecoveryPoints());
+ if (replace) {
+ verifyNoInteractions(fixture.sns);
+ } else {
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ verify(fixture.sns).publish(eq(TOPIC), isNull(), isNull(), anyString(),
+ eq("Notification from AWS Backup"), anyMap(), eq(REGION)));
+ }
+ }
+ }
+ }
+
+ @Test
+ void resumedCompletionPreservesSnsPayloadAndAttributesAndFiltersOtherEvents() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ fixture.service.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_FAILED"));
+ BackupJob quiet = fixture.startRunning();
+ quiet.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);
+ fixture.service.shutdown();
+ fixture.service = fixture.newService();
+ fixture.service.resumeJobs();
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ assertEquals("COMPLETED", fixture.service.describeBackupJob(quiet.getBackupJobId()).getState()));
+ // The terminal state is visible before notify returns; wait for the completion monitor
+ // before changing the event filter for the next job.
+ synchronized (fixture.vaultMonitor()) {
+ verifyNoInteractions(fixture.sns);
+ fixture.service.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_COMPLETED"));
+ }
+ BackupJob job = fixture.startRunning();
+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);
+ fixture.service.shutdown();
+ fixture.service = fixture.newService();
+ fixture.service.resumeJobs();
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<Map<String, MessageAttributeValue>> attrs = ArgumentCaptor.forClass(Map.class);
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ verify(fixture.sns).publish(eq(TOPIC), isNull(), isNull(), eq(
+ "An AWS Backup job was completed successfully. Recovery point ARN: "
+ + job.getRecoveryPointArn() + ". Resource ARN : " + job.getResourceArn()
+ + ". BackupJob ID : " + job.getBackupJobId()),
+ eq("Notification from AWS Backup"), attrs.capture(), eq(REGION)));
+ Map<String, MessageAttributeValue> expected = Map.of(
+ "EventType", new MessageAttributeValue("BACKUP_JOB", "String"),
+ "State", new MessageAttributeValue("COMPLETED", "String"),
+ "AccountId", new MessageAttributeValue(ACCOUNT, "String"),
+ "Id", new MessageAttributeValue(job.getBackupJobId(), "String"),
+ "StartTime", new MessageAttributeValue(Instant.ofEpochSecond(job.getCreationDate()).toString(), "String"));
+ assertEquals(fixture.mapper.valueToTree(expected), fixture.mapper.valueToTree(attrs.getValue()));
+ fixture.service.resumeJobs();
+ fixture.complete(job);
+ verifyNoMoreInteractions(fixture.sns);
+ }
+ }
+
+ @Test
+ void resumedPublishFailureDoesNotUndoCompletionOrPoint() {
+ try (Fixture fixture = new Fixture()) {
+ fixture.service.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_COMPLETED"));
+ doThrow(new IllegalStateException("offline topic")).when(fixture.sns)
+ .publish(anyString(), isNull(), isNull(), anyString(), anyString(), anyMap(), anyString());
+ BackupJob job = fixture.startRunning();
+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);
+ fixture.service.shutdown();
+ fixture.service = fixture.newService();
+ fixture.service.resumeJobs();
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> {
+ verify(fixture.sns).publish(eq(TOPIC), isNull(), isNull(), anyString(), anyString(), anyMap(), eq(REGION));
+ assertEquals("COMPLETED", fixture.service.describeBackupJob(job.getBackupJobId()).getState());
+ assertEquals(1, fixture.points.keys().size());
+ assertEquals(1, fixture.vault().getNumberOfRecoveryPoints());
+ });
+ }
+ }
+
+ @Test
+ void terminalJobsCannotBeStoppedOrCompletedAgain() throws Exception {
+ for (String state : List.of("COMPLETED", "ABORTED", "FAILED", "EXPIRED", "PARTIAL")) {
+ try (Fixture fixture = new Fixture()) {
+ BackupJob job = fixture.legacy(state, Instant.now().getEpochSecond());
+ AwsException error = assertThrows(AwsException.class, () -> fixture.service.stopBackupJob("legacy"));
+ assertEquals("InvalidRequestException", error.getErrorCode());
+ fixture.complete(job);
+ fixture.service.resumeJobs();
+ assertEquals(state, fixture.service.describeBackupJob("legacy").getState());
+ assertNull(job.getRecoveryPointArn());
+ assertTrue(fixture.points.keys().isEmpty());
+ verifyNoInteractions(fixture.sns);
+ }
+ }
+ }
+
+ @Test
+ void persistentJsonStoresReloadCapturedIdentityBeforeResumingOriginalOrReplacement(@TempDir Path directory)
+ throws Exception {
+ for (boolean replace : List.of(false, true)) {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ Path path = directory.resolve(Boolean.toString(replace));
+ when(config.storage().persistentPath()).thenReturn(path.toString());
+ when(config.defaultAccountId()).thenReturn(ACCOUNT);
+ when(config.services().backup().jobCompletionDelaySeconds()).thenReturn(3600);
+ ServiceConfigAccess access = mock(ServiceConfigAccess.class);
+ when(access.storageMode("backup")).thenReturn("persistent");
+ StorageFactory firstStorage = new StorageFactory(config, access);
+ RegionResolver resolver = new RegionResolver("us-east-1", ACCOUNT);
+ ObjectMapper mapper = new ObjectMapper();
+ SnsService sns = mock(SnsService.class);
+ BackupService first = new BackupService(firstStorage, config, resolver, mapper, sns);
+ String jobId;
+ String capturedId;
+ long capturedDate;
+ try {
+ BackupVault vault = first.createBackupVault(VAULT, null, null, Map.of(), REGION);
+ capturedId = vault.getVaultId();
+ capturedDate = vault.getCreationDate();
+ BackupJob job = first.startBackupJob(VAULT, "arn:aws:s3:::persisted", "role", null, REGION);
+ jobId = job.getBackupJobId();
+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);
+ AccountAwareStorageBackend<BackupJob> jobs = firstStorage.create("backup", "backup-jobs.json",
+ new TypeReference<Map<String, BackupJob>>() {});
+ jobs.put(jobId, job);
+ } finally {
+ first.shutdown();
+ firstStorage.shutdownAll();
+ }
+ JsonNode disk = mapper.readTree(path.resolve("backup-jobs.json").toFile()).path(ACCOUNT + "/" + jobId);
+ assertEquals(capturedId, disk.path("FlociVaultId").asText());
+ StorageFactory reloadedStorage = new StorageFactory(config, access);
+ BackupService restarted = new BackupService(reloadedStorage, config, resolver, mapper, sns);
+ try {
+ assertEquals(capturedId, restarted.describeBackupJob(jobId).getVaultId());
+ assertEquals(capturedId, restarted.describeBackupVault(VAULT, REGION).getVaultId());
+ if (replace) {
+ restarted.deleteBackupVault(VAULT, REGION);
+ BackupVault replacement = restarted.createBackupVault(VAULT, null, null, Map.of(), REGION);
+ replacement.setCreationDate(capturedDate);
+ assertNotEquals(capturedId, replacement.getVaultId());
+ }
+ restarted.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_COMPLETED"));
+ restarted.resumeJobs();
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertEquals(
+ replace ? "FAILED" : "COMPLETED", restarted.describeBackupJob(jobId).getState()));
+ assertEquals(replace ? 0 : 1, restarted.listRecoveryPointsByBackupVault(VAULT, REGION).size());
+ assertEquals(replace ? 0 : 1, restarted.describeBackupVault(VAULT, REGION).getNumberOfRecoveryPoints());
+ assertEquals(capturedId, restarted.describeBackupJob(jobId).getVaultId());
+ if (replace) {
+ verifyNoInteractions(sns);
+ } else {
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ verify(sns).publish(eq(TOPIC), isNull(), isNull(), anyString(), anyString(), anyMap(), eq(REGION)));
+ }
+ } finally {
+ restarted.shutdown();
+ reloadedStorage.shutdownAll();
+ }
+ }
+ }
+
+ @Test
+ void capturedIdStillRequiresOriginalArnAfterJsonRoundtrip() throws Exception {
+ for (boolean matchingArn : List.of(true, false)) {
+ try (Fixture fixture = new Fixture()) {
+ BackupJob job = fixture.startRunning();
+ String capturedId = job.getVaultId();
+ job = fixture.mapper.readValue(fixture.mapper.writeValueAsString(job), BackupJob.class);
+ if (!matchingArn) {
+ job.setBackupVaultArn(job.getBackupVaultArn() + "-different");
+ }
+ fixture.jobs.put(job.getBackupJobId(), job);
+ fixture.complete(job);
+ assertEquals(matchingArn ? "COMPLETED" : "FAILED", job.getState());
+ assertEquals(matchingArn ? 1 : 0, fixture.points.keys().size());
+ assertEquals(matchingArn ? 1 : 0, fixture.vault().getNumberOfRecoveryPoints());
+ assertEquals(capturedId, job.getVaultId());
+ assertEquals(fixture.vault().getVaultId(), job.getVaultId());
+ }
+ }
+ }
+
+ @Test
+ void legacyMissingCompletionDeadlineUsesCreationPlusConfiguredDelay() {
+ try (Fixture fixture = new Fixture()) {
+ long now = Instant.now().getEpochSecond();
+ fixture.vault().setCreationDate(now - 7200);
+ BackupJob job = fixture.legacy("CREATED", now - 3601);
+ job.setExpectedCompletionDate(null);
+ fixture.service.resumeJobs();
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ assertEquals("COMPLETED", fixture.service.describeBackupJob("legacy").getState()));
+ assertEquals(1, fixture.points.keys().size());
+ assertEquals(1, fixture.vault().getNumberOfRecoveryPoints());
+ assertTrue(job.getRecoveryPointArn().contains(":" + REGION + ":" + ACCOUNT + ":"));
+ }
+ }
+
+ private static class Fixture implements AutoCloseable {
+ private final Map<String, AccountAwareStorageBackend<?>> stores = new HashMap<>();
+ private final StorageFactory storage = mock(StorageFactory.class);
+ private final EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ private final RegionResolver resolver = new RegionResolver("us-east-1", ACCOUNT);
+ private final ObjectMapper mapper = new ObjectMapper();
+ private final SnsService sns = mock(SnsService.class);
+ private final AccountAwareStorageBackend<BackupVault> vaults;
+ private final AccountAwareStorageBackend<BackupJob> jobs;
+ private final AccountAwareStorageBackend<RecoveryPoint> points;
+ private BackupService service;
+
+ Fixture() {
+ when(storage.create(eq("backup"), anyString(), any())).thenAnswer(call ->
+ stores.computeIfAbsent(call.getArgument(1), ignored ->
+ spy(AccountAwareStorageBackend.inMemory(ACCOUNT))));
+ when(config.services().backup().jobCompletionDelaySeconds()).thenReturn(3600);
+ service = newService();
+ vaults = store("backup-vaults.json");
+ jobs = store("backup-jobs.json");
+ points = store("backup-recovery-points.json");
+ service.createBackupVault(VAULT, null, null, Map.of(), REGION);
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> AccountAwareStorageBackend<T> store(String name) {
+ return (AccountAwareStorageBackend<T>) stores.get(name);
+ }
+
+ private BackupService newService() {
+ return new BackupService(storage, config, resolver, mapper, sns);
+ }
+
+ private BackupVault vault() {
+ return service.describeBackupVault(VAULT, REGION);
+ }
+
+ private BackupJob startRunning() {
+ BackupJob job = service.startBackupJob(VAULT, "arn:aws:s3:::review-bucket", "role", null, REGION);
+ job.setState("RUNNING");
+ jobs.put(job.getBackupJobId(), job);
+ return job;
+ }
+
+ private BackupJob legacy(String state, long creationDate) {
+ BackupJob job = new BackupJob();
+ job.setBackupJobId("legacy");
+ job.setBackupVaultName(VAULT);
+ job.setBackupVaultArn(vault().getBackupVaultArn());
+ job.setResourceArn("arn:aws:s3:::review-bucket");
+ job.setAccountId(ACCOUNT);
+ job.setState(state);
+ job.setCreationDate(creationDate);
+ job.setStartBy(Instant.now().getEpochSecond() + 3600);
+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);
+ jobs.put(job.getBackupJobId(), job);
+ return job;
+ }
+
+ private void complete(BackupJob job) throws Exception {
+ Method method = BackupService.class.getDeclaredMethod("completeJob", String.class, String.class, String.class);
+ method.setAccessible(true);
+ method.invoke(service, job.getBackupJobId(), VAULT, REGION);
+ }
+
+ private Object vaultMonitor() throws Exception {
+ Field field = BackupService.class.getDeclaredField("vaultLocks");
+ field.setAccessible(true);
+ Map<?, ?> locks = (Map<?, ?>) field.get(service);
+ return locks.entrySet().stream()
+ .filter(entry -> entry.getKey().toString().endsWith(REGION + ":" + VAULT))
+ .map(Map.Entry::getValue).findFirst().orElseThrow();
+ }
+
+ @Override
+ public void close() {
+ service.shutdown();
+ }
+ }
+}