Light, fluffy, and always free - The AWS Local Emulator alternative
fix(backup): resume unfinished jobs after restart (#5721)
Daniël van Ziel6d96849d2afa
diff --git a/docs/services/backup.md b/docs/services/backup.mdindex ca6d9b0106..ea0966809e 100644--- a/docs/services/backup.md+++ b/docs/services/backup.md@@ -110,6 +110,28 @@ FLOCI_SERVICES_BACKUP_JOB_COMPLETION_DELAY_SECONDS=3Use a shorter delay (e.g. `1`) in test environments to speed up job-completion assertions.+On startup, persisted `CREATED`, `RUNNING` and `ABORTING` jobs resume in their owning+account and original region. Existing completion deadlines are retained; older jobs without+one use their creation time plus the configured delay. A `CREATED` job whose start window+has elapsed becomes `EXPIRED`. Terminal jobs are not restarted and cannot be stopped.++Completion writes a stable recovery point before marking the job `COMPLETED`. Recovery+after an interrupted completion reuses that point without changing its original creation+date, and reconciles the vault count only on this retry path. A new point increments the+count without scanning the recovery-point store. Completion and recovery-point deletion+share the vault monitor so retention checks, deletion and count updates stay consistent.++New jobs persist the vault's internal `FlociVaultId` before scheduling.+Completion requires the original vault ARN and captured id, so deleting and recreating a+vault with the same name and ARN, even within one second, fails the old job without writing+a point or notifying the replacement. Legacy jobs without `FlociVaultId` accept only a vault+no newer than the job; historical dates cannot prove that a same-second vault is the original+incarnation. Recovery never+fills a missing job id from the currently resolved vault. The internal identity field stays+in persisted JSON but is omitted from `DescribeBackupJob` and `ListBackupJobs` responses.+Successful resumed completions use the existing SNS notification path and event filter;+publish failures do not undo the job or recovery point.+## Supported Resource Types`GetSupportedResourceTypes` returns the following resource type codes:diff --git a/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java b/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.javaindex 41f0be9d78..6788ef373b 100644--- a/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java+++ b/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java@@ -416,7 +416,7 @@ public class BackupController {@GET@Path("/backup-jobs/{backupJobId}")public Response describeBackupJob(@PathParam("backupJobId") String jobId) {- return Response.ok(service.describeBackupJob(jobId)).build();+ return Response.ok(jobJson(service.describeBackupJob(jobId))).build();}@POST@@ -435,10 +435,16 @@ public class BackupController {List<BackupJob> jobs = service.listBackupJobs(byVaultName, byState, byResourceArn, byResourceType);ObjectNode out = objectMapper.createObjectNode();ArrayNode list = out.putArray("BackupJobs");- jobs.forEach(list::addPOJO);+ jobs.forEach(job -> list.add(jobJson(job)));return Response.ok(out).build();}+ private ObjectNode jobJson(BackupJob job) {+ ObjectNode node = objectMapper.valueToTree(job);+ node.remove(BackupVault.VAULT_ID_FIELD);+ return node;+ }+// ── Recovery Point ─────────────────────────────────────────────────────────@GETdiff --git a/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java b/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.javaindex daad128344..6edcc9d61b 100644--- a/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java+++ b/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java@@ -8,11 +8,21 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;-import io.github.hectorvent.floci.services.backup.model.*;+import io.github.hectorvent.floci.services.backup.model.BackupJob;+import io.github.hectorvent.floci.services.backup.model.BackupPlan;+import io.github.hectorvent.floci.services.backup.model.BackupRule;+import io.github.hectorvent.floci.services.backup.model.BackupSelection;+import io.github.hectorvent.floci.services.backup.model.BackupVault;+import io.github.hectorvent.floci.services.backup.model.BackupVaultNotifications;+import io.github.hectorvent.floci.services.backup.model.Lifecycle;+import io.github.hectorvent.floci.services.backup.model.RecoveryPoint;import io.github.hectorvent.floci.services.sns.SnsService;import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;+import io.quarkus.runtime.Startup;+import jakarta.annotation.PostConstruct;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -21,13 +31,19 @@ import org.jboss.logging.Logger;import java.time.Instant;import java.time.format.DateTimeFormatter;import java.time.temporal.ChronoUnit;-import java.util.*;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.Executors;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.TimeUnit;@ApplicationScoped+@Startuppublic class BackupService {private static final Logger LOG = Logger.getLogger(BackupService.class);@@ -40,7 +56,7 @@ public class BackupService {private final StorageBackend<String, BackupVault> vaultStore;private final StorageBackend<String, BackupPlan> planStore;private final StorageBackend<String, BackupSelection> selectionStore;- private final StorageBackend<String, BackupJob> jobStore;+ private final AccountAwareStorageBackend<BackupJob> jobStore;private final StorageBackend<String, RecoveryPoint> recoveryStore;private final StorageBackend<String, String> accessPolicyStore;private final StorageBackend<String, BackupVaultNotifications> notificationStore;@@ -86,6 +102,53 @@ public class BackupService {scheduler.shutdownNow();}+ @PostConstruct+ void resumeJobs() {+ for (AccountAwareStorageBackend.AccountEntry<BackupJob> entry : jobStore.scanAllAccountEntries(k -> true)) {+ BackupJob job = entry.value();+ if (!List.of("CREATED", "RUNNING", "ABORTING").contains(job.getState())) {+ continue;+ }+ String region = AwsArnUtils.regionOrDefault(job.getBackupVaultArn(), regionResolver.getDefaultRegion());+ if ("ABORTING".equals(job.getState())) {+ scheduler.execute(() -> RequestScopes.runAs(entry.accountId(), region, () -> abortJob(entry.key())));+ } else {+ scheduleJob(entry.key(), job, entry.accountId(), region);+ }+ }+ }++ private void scheduleJob(String jobId, BackupJob job, String accountId, String region) {+ long now = Instant.now().getEpochSecond();+ if ("CREATED".equals(job.getState()) && job.getStartBy() != null && job.getStartBy() < now) {+ scheduler.execute(() -> RequestScopes.runAs(accountId, region, () -> {+ synchronized (lockFor(jobLockKey(jobId))) {+ jobStore.get(jobId).ifPresent(current -> {+ if ("CREATED".equals(current.getState())) {+ current.setState("EXPIRED");+ current.setStatusMessage("The job did not start before its start window expired");+ current.setCompletionDate(Instant.now().getEpochSecond());+ jobStore.put(jobId, current);+ }+ });+ }+ }));+ return;+ }+ long completion = job.getExpectedCompletionDate() == null+ ? job.getCreationDate() + jobCompletionDelaySeconds : job.getExpectedCompletionDate();+ scheduler.schedule(() -> RequestScopes.runAs(accountId, region,+ () -> transitionJob(jobId, job.getBackupVaultName(), region)),+ Math.max(0, Math.min(job.getCreationDate() + 1, completion) - now), TimeUnit.SECONDS);+ scheduler.schedule(() -> RequestScopes.runAs(accountId, region,+ () -> completeJob(jobId, job.getBackupVaultName(), region)),+ Math.max(0, completion - now), TimeUnit.SECONDS);+ }++ private String jobLockKey(String jobId) {+ return "job:" + regionResolver.getAccountId() + ":" + jobId;+ }+// ── Vault ──────────────────────────────────────────────────────────────────public BackupVault createBackupVault(String vaultName, String encryptionKeyArn,@@ -653,6 +716,7 @@ public class BackupService {job.setBackupJobId(jobId);job.setBackupVaultName(vaultName);job.setBackupVaultArn(vault.getBackupVaultArn());+ job.setVaultId(vault.getVaultId());job.setResourceArn(resourceArn);job.setResourceType(inferResourceType(resourceArn));job.setIamRoleArn(iamRoleArn);@@ -664,16 +728,7 @@ public class BackupService {String accountId = regionResolver.getAccountId();job.setAccountId(accountId);jobStore.put(jobId, job);- // Pinned now, not looked up by name at completion: if this vault is deleted and the name- // reused while the job runs, the job must not notify the replacement vault's topic.- String notificationKey = subResourceKey(region, vault);-- // The scheduler thread has no request context; run in the caller's account- // so the account-aware stores and buildArn resolve the right partition.- scheduler.schedule(() -> RequestScopes.runAs(accountId,- () -> transitionJob(jobId, vaultName, region)), 1, TimeUnit.SECONDS);- scheduler.schedule(() -> RequestScopes.runAs(accountId,- () -> completeJob(jobId, vaultName, region, notificationKey)), jobCompletionDelaySeconds, TimeUnit.SECONDS);+ scheduleJob(jobId, job, accountId, region);return job;}@@ -684,17 +739,20 @@ public class BackupService {}public void stopBackupJob(String jobId) {- BackupJob job = describeBackupJob(jobId);- String state = job.getState();- if ("COMPLETED".equals(state) || "ABORTED".equals(state) || "FAILED".equals(state)) {- throw new AwsException("InvalidRequestException",- "Job cannot be stopped in state: " + state, 400);+ synchronized (lockFor(jobLockKey(jobId))) {+ BackupJob job = describeBackupJob(jobId);+ String state = job.getState();+ if (List.of("COMPLETED", "ABORTED", "FAILED", "EXPIRED", "PARTIAL").contains(state)) {+ throw new AwsException("InvalidRequestException",+ "Job cannot be stopped in state: " + state, 400);+ }+ job.setState("ABORTING");+ job.setStatusMessage("Job stop requested");+ jobStore.put(jobId, job);+ String accountId = regionResolver.getAccountId();+ String region = AwsArnUtils.regionOrDefault(job.getBackupVaultArn(), regionResolver.getDefaultRegion());+ scheduler.schedule(() -> RequestScopes.runAs(accountId, region, () -> abortJob(jobId)), 1, TimeUnit.SECONDS);}- job.setState("ABORTING");- job.setStatusMessage("Job stop requested");- jobStore.put(jobId, job);- String accountId = regionResolver.getAccountId();- scheduler.schedule(() -> RequestScopes.runAs(accountId, () -> abortJob(jobId)), 1, TimeUnit.SECONDS);}public List<BackupJob> listBackupJobs(String byVaultName, String byState,@@ -743,20 +801,22 @@ public class BackupService {* passes would be describing AWS behaviour that needs a specific permission to reproduce.*/public void deleteRecoveryPoint(String vaultName, String recoveryPointArn, String region) {- BackupVault vault = describeBackupVault(vaultName, region);- RecoveryPoint rp = describeRecoveryPoint(vaultName, recoveryPointArn, region);- if (vault.isLocked() && vault.getMinRetentionDays() != null) {- long ageDays = ChronoUnit.DAYS.between(- Instant.ofEpochSecond(rp.getCreationDate()), Instant.now());- if (ageDays < vault.getMinRetentionDays()) {- throw new AwsException("InvalidRequestException",- "Recovery point is protected by a vault lock until its minimum retention of "- + vault.getMinRetentionDays() + " day(s) has elapsed: " + recoveryPointArn,- 400);+ synchronized (lockFor(vaultKey(region, vaultName))) {+ BackupVault vault = describeBackupVault(vaultName, region);+ RecoveryPoint rp = describeRecoveryPoint(vaultName, recoveryPointArn, region);+ if (vault.isLocked() && vault.getMinRetentionDays() != null) {+ long ageDays = ChronoUnit.DAYS.between(+ Instant.ofEpochSecond(rp.getCreationDate()), Instant.now());+ if (ageDays < vault.getMinRetentionDays()) {+ throw new AwsException("InvalidRequestException",+ "Recovery point is protected by a vault lock until its minimum retention of "+ + vault.getMinRetentionDays() + " day(s) has elapsed: " + recoveryPointArn,+ 400);+ }}+ recoveryStore.delete(recoveryPointArn);+ decrementVaultCount(vaultName, region);}- recoveryStore.delete(recoveryPointArn);- decrementVaultCount(vaultName, region);}// ── Tags ───────────────────────────────────────────────────────────────────@@ -782,24 +842,28 @@ public class BackupService {// ── Private helpers ────────────────────────────────────────────────────────private void transitionJob(String jobId, String vaultName, String region) {- jobStore.get(jobId).ifPresent(job -> {- if ("CREATED".equals(job.getState())) {- job.setState("RUNNING");- job.setPercentDone("50.0");- jobStore.put(jobId, job);- }- });+ synchronized (lockFor(jobLockKey(jobId))) {+ jobStore.get(jobId).ifPresent(job -> {+ if ("CREATED".equals(job.getState())) {+ job.setState("RUNNING");+ job.setPercentDone("50.0");+ jobStore.put(jobId, job);+ }+ });+ }}- private void completeJob(String jobId, String vaultName, String region, String notificationKey) {+ private void completeJob(String jobId, String vaultName, String region) {// Under the vault-name monitor, so the vault cannot be deleted or recreated between the// incarnation check and the recovery point landing in it.- synchronized (lockFor(vaultKey(region, vaultName))) {- completeJobLocked(jobId, vaultName, region, notificationKey);+ synchronized (lockFor(jobLockKey(jobId))) {+ synchronized (lockFor(vaultKey(region, vaultName))) {+ completeJobLocked(jobId, vaultName, region);+ }}}- private void completeJobLocked(String jobId, String vaultName, String region, String notificationKey) {+ private void completeJobLocked(String jobId, String vaultName, String region) {jobStore.get(jobId).ifPresent(job -> {if ("RUNNING".equals(job.getState())) {long now = Instant.now().getEpochSecond();@@ -807,9 +871,10 @@ public class BackupService {// vault holds the name now. A vault deleted mid-job (allowed: it held no recovery// point yet) and recreated unlocked would otherwise receive a recovery point its// creator's lock was meant to protect, deletable inside that lock's retention.- boolean sameVault = vaultStore.get(vaultKey(region, vaultName))- .map(v -> subResourceKey(region, v).equals(notificationKey))- .orElse(false);+ Optional<BackupVault> vault = vaultStore.get(vaultKey(region, vaultName));+ boolean sameVault = vault.filter(v -> v.getBackupVaultArn().equals(job.getBackupVaultArn())+ && (job.getVaultId() != null ? job.getVaultId().equals(v.getVaultId())+ : v.getCreationDate() <= job.getCreationDate())).isPresent();if (!sameVault) {job.setState("FAILED");job.setStatusMessage("Backup vault was deleted before the job completed: " + vaultName);@@ -817,9 +882,31 @@ public class BackupService {jobStore.put(jobId, job);return;}- String rpArn = regionResolver.buildArn("backup", region,- "recovery-point:" + UUID.randomUUID());-+ BackupVault currentVault = vault.orElseThrow();+ String notificationKey = subResourceKey(region, currentVault);+ String rpArn = job.getRecoveryPointArn() != null ? job.getRecoveryPointArn()+ : regionResolver.buildArn("backup", region, "recovery-point:" + jobId);+ if (recoveryStore.get(rpArn).isEmpty()) {+ RecoveryPoint rp = new RecoveryPoint();+ rp.setRecoveryPointArn(rpArn);+ rp.setBackupVaultName(vaultName);+ rp.setBackupVaultArn(job.getBackupVaultArn());+ rp.setResourceArn(job.getResourceArn());+ rp.setResourceType(job.getResourceType());+ rp.setIamRoleArn(job.getIamRoleArn());+ rp.setStatus("COMPLETED");+ rp.setCreationDate(job.getCreationDate());+ rp.setCompletionDate(now);+ rp.setBackupSizeInBytes(0L);+ rp.setStorageClass("WARM");+ rp.setEncrypted(false);+ recoveryStore.put(rpArn, rp);+ currentVault.setNumberOfRecoveryPoints(currentVault.getNumberOfRecoveryPoints() + 1);+ } else {+ currentVault.setNumberOfRecoveryPoints((int) recoveryStore.scan(k -> true).stream()+ .filter(point -> currentVault.getBackupVaultArn().equals(point.getBackupVaultArn())).count());+ }+ vaultStore.put(vaultKey(region, vaultName), currentVault);job.setState("COMPLETED");job.setPercentDone("100.0");job.setCompletionDate(now);@@ -827,23 +914,6 @@ public class BackupService {job.setBackupSizeInBytes(0L);job.setBytesTransferred(0L);jobStore.put(jobId, job);-- RecoveryPoint rp = new RecoveryPoint();- rp.setRecoveryPointArn(rpArn);- rp.setBackupVaultName(vaultName);- rp.setBackupVaultArn(job.getBackupVaultArn());- rp.setResourceArn(job.getResourceArn());- rp.setResourceType(job.getResourceType());- rp.setIamRoleArn(job.getIamRoleArn());- rp.setStatus("COMPLETED");- rp.setCreationDate(job.getCreationDate());- rp.setCompletionDate(now);- rp.setBackupSizeInBytes(0L);- rp.setStorageClass("WARM");- rp.setEncrypted(false);- recoveryStore.put(rpArn, rp);-- incrementVaultCount(vaultName, region);LOG.infov("Backup job {0} completed, recovery point: {1}", jobId, rpArn);notifyJobCompleted(job, vaultName, region, notificationKey);}@@ -885,20 +955,15 @@ public class BackupService {}private void abortJob(String jobId) {- jobStore.get(jobId).ifPresent(job -> {- if ("ABORTING".equals(job.getState())) {- job.setState("ABORTED");- job.setCompletionDate(Instant.now().getEpochSecond());- jobStore.put(jobId, job);- }- });- }-- private void incrementVaultCount(String vaultName, String region) {- vaultStore.get(vaultKey(region, vaultName)).ifPresent(vault -> {- vault.setNumberOfRecoveryPoints(vault.getNumberOfRecoveryPoints() + 1);- vaultStore.put(vaultKey(region, vaultName), vault);- });+ synchronized (lockFor(jobLockKey(jobId))) {+ jobStore.get(jobId).ifPresent(job -> {+ if ("ABORTING".equals(job.getState())) {+ job.setState("ABORTED");+ job.setCompletionDate(Instant.now().getEpochSecond());+ jobStore.put(jobId, job);+ }+ });+ }}private void decrementVaultCount(String vaultName, String region) {diff --git a/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupJob.java b/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupJob.javaindex d5dd8d7b50..341ddfee9a 100644--- a/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupJob.java+++ b/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupJob.java@@ -19,6 +19,9 @@ public class BackupJob {@JsonProperty("BackupVaultArn")private String backupVaultArn;+ @JsonProperty("FlociVaultId")+ private String vaultId;+@JsonProperty("RecoveryPointArn")private String recoveryPointArn;@@ -72,6 +75,9 @@ public class BackupJob {public String getBackupVaultArn() { return backupVaultArn; }public void setBackupVaultArn(String backupVaultArn) { this.backupVaultArn = backupVaultArn; }+ public String getVaultId() { return vaultId; }+ public void setVaultId(String vaultId) { this.vaultId = vaultId; }+public String getRecoveryPointArn() { return recoveryPointArn; }public void setRecoveryPointArn(String recoveryPointArn) { this.recoveryPointArn = recoveryPointArn; }diff --git a/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.javaindex ad7b73cbb2..8a93bebdae 100644--- a/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java@@ -1,8 +1,12 @@package io.github.hectorvent.floci.services.backup;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.services.backup.model.BackupJob;import io.quarkus.test.junit.QuarkusTest;import io.restassured.path.json.JsonPath;import io.restassured.specification.RequestSpecification;+import jakarta.inject.Inject;import org.junit.jupiter.api.MethodOrderer;import org.junit.jupiter.api.Order;import org.junit.jupiter.api.Test;@@ -31,6 +35,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals;@TestMethodOrder(MethodOrderer.OrderAnnotation.class)class BackupIntegrationTest {+ @Inject BackupService backupService;+ @Inject ObjectMapper mapper;+private static final String AUTH = "AWS4-HMAC-SHA256 Credential=test/20260205/us-east-1/backup/aws4_request";private static final String VAULT_NAME = "test-vault";private static final String IAM_ROLE = "arn:aws:iam::000000000000:role/backup-role";@@ -1941,4 +1948,45 @@ class BackupIntegrationTest {.body("NumberOfRecoveryPoints", equalTo(0));}+ @Test+ @Order(201)+ void describeAndListJobsHideCapturedIdentityWithoutRemovingStoredOrPublicFields() {+ String vault = "job-api-identity-vault";+ given().header("Authorization", AUTH).contentType("application/json").body("{}")+ .when().put("/backup-vaults/" + vault).then().statusCode(200);+ String jid = given().header("Authorization", AUTH).contentType("application/json")+ .body("""+ {"BackupVaultName":"%s","ResourceArn":"%s","IamRoleArn":"%s"}+ """.formatted(vault, RESOURCE_ARN, IAM_ROLE))+ .when().put("/backup-jobs").then().statusCode(200).extract().path("BackupJobId");+ BackupJob stored = backupService.describeBackupJob(jid);+ String capturedId = backupService.describeBackupVault(vault, "us-east-1").getVaultId();+ assertEquals(capturedId, stored.getVaultId());+ JsonNode before = mapper.valueToTree(stored);+ assertEquals(capturedId, before.path("FlociVaultId").asText());+ given().header("Authorization", AUTH)+ .when().get("/backup-jobs/" + jid).then().statusCode(200)+ .body("$", not(hasKey("FlociVaultId")))+ .body("BackupJobId", equalTo(jid))+ .body("BackupVaultName", equalTo(vault))+ .body("BackupVaultArn", equalTo(stored.getBackupVaultArn()))+ .body("ResourceArn", equalTo(RESOURCE_ARN))+ .body("CreationDate", notNullValue())+ .body("State", notNullValue());+ given().header("Authorization", AUTH).queryParam("byBackupVaultName", vault)+ .when().get("/backup-jobs/").then().statusCode(200)+ .body("BackupJobs", hasSize(1))+ .body("BackupJobs[0]", not(hasKey("FlociVaultId")))+ .body("BackupJobs[0].BackupJobId", equalTo(jid))+ .body("BackupJobs[0].BackupVaultName", equalTo(vault))+ .body("BackupJobs[0].ResourceArn", equalTo(RESOURCE_ARN))+ .body("BackupJobs[0].CreationDate", notNullValue())+ .body("BackupJobs[0].State", notNullValue());+ JsonNode after = mapper.valueToTree(backupService.describeBackupJob(jid));+ assertEquals(before.path("FlociVaultId"), after.path("FlociVaultId"));+ awaitJobCompleted(jid);+ String point = backupService.describeBackupJob(jid).getRecoveryPointArn();+ backupService.deleteRecoveryPoint(vault, point, "us-east-1");+ backupService.deleteBackupVault(vault, "us-east-1");+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryIntegrationTest.javanew file mode 100644index 0000000000..bb02e0ac86--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryIntegrationTest.java@@ -0,0 +1,79 @@+package io.github.hectorvent.floci.services.backup;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.backup.model.BackupJob;+import io.github.hectorvent.floci.services.sns.SnsService;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.time.Instant;+import java.util.List;+import java.util.Map;+import java.util.UUID;++import static org.awaitility.Awaitility.await;+import static org.junit.jupiter.api.Assertions.*;++@QuarkusTest+class BackupJobRecoveryIntegrationTest {+ @Inject StorageFactory storage;+ @Inject EmulatorConfig config;+ @Inject RegionResolver resolver;+ @Inject ObjectMapper mapper;+ @Inject SnsService sns;++ @Test+ void restartRestoresJobsInEveryAccountAndTheirOriginalRegion() {+ String vaultName = "restart-" + UUID.randomUUID();+ List<String> accounts = List.of("811111111111", "822222222222");+ BackupService first = new BackupService(storage, config, resolver, mapper, sns);+ AccountAwareStorageBackend<BackupJob> jobs = storage.create("backup", "backup-jobs.json",+ new TypeReference<Map<String, BackupJob>>() {});+ try {+ for (String account : accounts) {+ RequestScopes.runAs(account, "eu-west-1", () -> {+ first.createBackupVault(vaultName, null, null, Map.of(), "eu-west-1");+ BackupJob job = first.startBackupJob(vaultName, "arn:aws:s3:::bucket", "role", null, "eu-west-1");+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);+ jobs.put(job.getBackupJobId(), job);+ });+ }+ } finally {+ first.shutdown();+ }+ BackupService restarted = new BackupService(storage, config, resolver, mapper, sns);+ try {+ restarted.resumeJobs();+ for (String account : accounts) {+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> RequestScopes.runAs(account, () -> {+ assertEquals("COMPLETED", restarted.listBackupJobs(vaultName, null, null, null).getFirst().getState());+ assertEquals(1, restarted.listRecoveryPointsByBackupVault(vaultName, "eu-west-1").size());+ String arn = restarted.listRecoveryPointsByBackupVault(vaultName, "eu-west-1")+ .getFirst().getRecoveryPointArn();+ assertTrue(arn.contains(":eu-west-1:" + account + ":"));+ assertEquals(1, restarted.describeBackupVault(vaultName, "eu-west-1").getNumberOfRecoveryPoints());+ assertEquals(restarted.describeBackupVault(vaultName, "eu-west-1").getVaultId(),+ restarted.listBackupJobs(vaultName, null, null, null).getFirst().getVaultId());+ }));+ }+ } finally {+ restarted.shutdown();+ for (String account : accounts) {+ RequestScopes.runAs(account, () -> {+ restarted.listRecoveryPointsByBackupVault(vaultName, "eu-west-1").forEach(point ->+ restarted.deleteRecoveryPoint(vaultName, point.getRecoveryPointArn(), "eu-west-1"));+ restarted.listBackupJobs(vaultName, null, null, null).forEach(job -> jobs.delete(job.getBackupJobId()));+ restarted.deleteBackupVault(vaultName, "eu-west-1");+ });+ }+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryTest.java b/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryTest.javanew file mode 100644index 0000000000..237eb1021e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/backup/BackupJobRecoveryTest.java@@ -0,0 +1,655 @@+package io.github.hectorvent.floci.services.backup;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.ServiceConfigAccess;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.backup.model.BackupJob;+import io.github.hectorvent.floci.services.backup.model.BackupVault;+import io.github.hectorvent.floci.services.backup.model.RecoveryPoint;+import io.github.hectorvent.floci.services.sns.SnsService;+import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;+import org.mockito.ArgumentCaptor;++import java.lang.reflect.Field;+import java.lang.reflect.Method;+import java.nio.file.Path;+import java.time.Duration;+import java.time.Instant;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.FutureTask;+import java.util.concurrent.TimeUnit;++import static org.awaitility.Awaitility.await;+import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.*;+import static org.mockito.Mockito.*;++class BackupJobRecoveryTest {+ private static final String ACCOUNT = "123456789012";+ private static final String REGION = "eu-west-1";+ private static final String VAULT = "review";+ private static final String TOPIC = "arn:aws:sns:eu-west-1:" + ACCOUNT + ":backup";++ @Test+ void restartResumesEachNonTerminalStateAndDoesNotDuplicateRecoveryPoints() {+ try (Fixture fixture = new Fixture()) {+ fixture.vault().setCreationDate(Instant.now().getEpochSecond() - 60);+ BackupJob original = fixture.service.startBackupJob(VAULT, "arn:aws:s3:::bucket", "role", null, REGION);+ fixture.service.shutdown();+ long now = Instant.now().getEpochSecond();+ original.setExpectedCompletionDate(now - 1);+ fixture.jobs.put(original.getBackupJobId(), original);+ for (String state : List.of("RUNNING", "ABORTING", "COMPLETED", "FAILED", "ABORTED")) {+ BackupJob job = fixture.legacy(state, now - 10);+ job.setBackupJobId(state);+ fixture.jobs.put(state, job);+ }+ fixture.jobs.delete("legacy");+ BackupJob expired = fixture.legacy("CREATED", now - 10);+ expired.setBackupJobId("expired");+ expired.setStartBy(now - 1);+ fixture.jobs.put("expired", expired);+ BackupJob missing = fixture.legacy("RUNNING", now - 10);+ missing.setBackupJobId("missing-vault");+ missing.setBackupVaultName("deleted");+ fixture.jobs.put("missing-vault", missing);+ BackupJob future = fixture.legacy("RUNNING", now - 10);+ future.setBackupJobId("future");+ future.setExpectedCompletionDate(now + 3600);+ fixture.jobs.put("future", future);+ fixture.jobs.delete("legacy");+ RecoveryPoint existing = new RecoveryPoint();+ existing.setBackupVaultName(VAULT);+ existing.setBackupVaultArn(fixture.vault().getBackupVaultArn());+ existing.setRecoveryPointArn("arn:aws:backup:" + REGION + ":" + ACCOUNT + ":recovery-point:RUNNING");+ existing.setCreationDate(now - 20);+ fixture.points.put(existing.getRecoveryPointArn(), existing);+ fixture.service = fixture.newService();+ fixture.service.resumeJobs();+ assertEquals("RUNNING", fixture.service.describeBackupJob("future").getState());+ assertEquals(now + 3600, fixture.service.describeBackupJob("future").getExpectedCompletionDate());+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> {+ assertEquals("COMPLETED", fixture.service.describeBackupJob(original.getBackupJobId()).getState());+ assertEquals("COMPLETED", fixture.service.describeBackupJob("RUNNING").getState());+ assertEquals("ABORTED", fixture.service.describeBackupJob("ABORTING").getState());+ assertEquals("EXPIRED", fixture.service.describeBackupJob("expired").getState());+ assertEquals("FAILED", fixture.service.describeBackupJob("missing-vault").getState());+ });+ fixture.service.resumeJobs();+ assertEquals(2, fixture.service.listRecoveryPointsByBackupVault(VAULT, REGION).size());+ assertEquals(2, fixture.vault().getNumberOfRecoveryPoints());+ assertEquals(now - 20, fixture.points.get(existing.getRecoveryPointArn()).orElseThrow().getCreationDate());+ assertTrue(fixture.service.listRecoveryPointsByBackupVault(VAULT, REGION).stream()+ .allMatch(point -> point.getRecoveryPointArn().contains(":" + REGION + ":" + ACCOUNT + ":")));+ for (String state : List.of("COMPLETED", "FAILED", "ABORTED")) {+ assertEquals(state, fixture.service.describeBackupJob(state).getState());+ assertNull(fixture.service.describeBackupJob(state).getRecoveryPointArn());+ }+ }+ }++ @Test+ void deletionWaitsForCompletionMonitorAndLeavesCountConsistent() throws Exception {+ try (Fixture fixture = new Fixture()) {+ BackupJob first = fixture.startRunning();+ fixture.complete(first);+ BackupJob second = fixture.startRunning();+ Object monitor = fixture.vaultMonitor();+ CountDownLatch entered = new CountDownLatch(1);+ FutureTask<Void> deletion = new FutureTask<>(() -> {+ entered.countDown();+ fixture.service.deleteRecoveryPoint(VAULT, first.getRecoveryPointArn(), REGION);+ return null;+ });+ Thread worker = new Thread(deletion, "backup-review-delete");+ boolean waited;+ try {+ synchronized (monitor) {+ worker.start();+ assertTrue(entered.await(5, TimeUnit.SECONDS));+ await().atMost(Duration.ofSeconds(5)).until(() ->+ worker.getState() == Thread.State.BLOCKED || deletion.isDone());+ waited = worker.getState() == Thread.State.BLOCKED && !deletion.isDone();+ fixture.complete(second);+ }+ deletion.get(5, TimeUnit.SECONDS);+ assertAll(+ () -> assertTrue(waited, "Point deletion must wait on completion's vault monitor"),+ () -> assertEquals(1, fixture.service.listRecoveryPointsByBackupVault(VAULT, REGION).size()),+ () -> assertEquals(1, fixture.vault().getNumberOfRecoveryPoints()),+ () -> assertThrows(AwsException.class,+ () -> fixture.service.deleteBackupVault(VAULT, REGION)));+ fixture.service.deleteRecoveryPoint(VAULT, second.getRecoveryPointArn(), REGION);+ assertTrue(fixture.service.listRecoveryPointsByBackupVault(VAULT, REGION).isEmpty());+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());+ fixture.service.deleteBackupVault(VAULT, REGION);+ assertThrows(AwsException.class, fixture::vault);+ } finally {+ worker.join(5000);+ assertFalse(worker.isAlive(), "Deletion worker must be cleaned up");+ }+ }+ }++ @Test+ void retentionReadPointDeleteAndCountReadWriteAllHoldCompletionMonitor() throws Exception {+ try (Fixture fixture = new Fixture()) {+ BackupJob job = fixture.startRunning();+ fixture.complete(job);+ Object monitor = fixture.vaultMonitor();+ RecoveryPoint point = fixture.points.get(job.getRecoveryPointArn()).orElseThrow();+ fixture.service.putBackupVaultLockConfiguration(VAULT, REGION, 1L, null, null);+ doAnswer(call -> {+ assertTrue(Thread.holdsLock(monitor), "Retention and count reads must hold the vault monitor");+ return call.callRealMethod();+ }).when(fixture.vaults).get(REGION + ":" + VAULT);+ doAnswer(call -> {+ assertTrue(Thread.holdsLock(monitor), "Point lookup for retention must hold the vault monitor");+ return call.callRealMethod();+ }).when(fixture.points).get(job.getRecoveryPointArn());+ doAnswer(call -> {+ assertTrue(Thread.holdsLock(monitor), "Point deletion must hold the vault monitor");+ return call.callRealMethod();+ }).when(fixture.points).delete(job.getRecoveryPointArn());+ doAnswer(call -> {+ assertTrue(Thread.holdsLock(monitor), "Count update must hold the vault monitor");+ return call.callRealMethod();+ }).when(fixture.vaults).put(eq(REGION + ":" + VAULT), any());+ AwsException protectedPoint = assertThrows(AwsException.class,+ () -> fixture.service.deleteRecoveryPoint(VAULT, job.getRecoveryPointArn(), REGION));+ assertEquals("InvalidRequestException", protectedPoint.getErrorCode());+ verify(fixture.points, never()).delete(anyString());+ point.setCreationDate(Instant.now().minus(Duration.ofDays(2)).getEpochSecond());+ fixture.service.deleteRecoveryPoint(VAULT, job.getRecoveryPointArn(), REGION);+ verify(fixture.points).delete(job.getRecoveryPointArn());+ doCallRealMethod().when(fixture.vaults).get(anyString());+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());+ fixture.service.deleteBackupVault(VAULT, REGION);+ }+ }++ @Test+ void brandNewPointIncrementsCountWithoutScanningRecoveryStore() throws Exception {+ try (Fixture fixture = new Fixture()) {+ doAnswer(call -> {+ BackupJob written = call.getArgument(1);+ if ("COMPLETED".equals(written.getState())) {+ assertTrue(fixture.points.get(written.getRecoveryPointArn()).isPresent(),+ "The point must be persisted before the terminal job write");+ assertTrue(Thread.holdsLock(fixture.vaultMonitor()));+ }+ return call.callRealMethod();+ }).when(fixture.jobs).put(anyString(), any());+ BackupJob first = fixture.startRunning();+ fixture.complete(first);+ BackupJob second = fixture.startRunning();+ fixture.complete(second);+ verify(fixture.points, never()).scan(any());+ assertEquals(2, fixture.vault().getNumberOfRecoveryPoints());+ assertEquals(2, fixture.points.keys().size());+ fixture.complete(first);+ assertEquals(2, fixture.vault().getNumberOfRecoveryPoints());+ verify(fixture.points, never()).scan(any());+ }+ }++ @Test+ void stablePointRetryRecountsAndRestartDoesNotDuplicateIt() throws Exception {+ try (Fixture fixture = new Fixture()) {+ BackupJob job = fixture.startRunning();+ fixture.complete(job);+ String pointArn = job.getRecoveryPointArn();+ RecoveryPoint original = fixture.points.get(pointArn).orElseThrow();+ original.setCreationDate(42L);+ fixture.vault().setNumberOfRecoveryPoints(9);+ job.setState("RUNNING");+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);+ fixture.jobs.put(job.getBackupJobId(), job);+ fixture.service.shutdown();+ clearInvocations(fixture.points);+ fixture.service = fixture.newService();+ fixture.service.resumeJobs();+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ assertEquals("COMPLETED", fixture.service.describeBackupJob(job.getBackupJobId()).getState()));+ verify(fixture.points, atLeastOnce()).scan(any());+ assertEquals(1, fixture.points.keys().size());+ assertEquals(1, fixture.vault().getNumberOfRecoveryPoints());+ assertEquals(42L, fixture.points.get(pointArn).orElseThrow().getCreationDate());+ assertEquals(pointArn, fixture.service.describeBackupJob(job.getBackupJobId()).getRecoveryPointArn());+ fixture.service.resumeJobs();+ fixture.complete(job);+ assertEquals(1, fixture.points.keys().size());+ assertEquals(1, fixture.vault().getNumberOfRecoveryPoints());+ }+ }++ @Test+ void newJobPersistsVaultIncarnationBeforeScheduling() {+ try (Fixture fixture = new Fixture()) {+ fixture.vault().setCreationDate(100L);+ doAnswer(call -> {+ BackupJob written = call.getArgument(1);+ assertEquals(fixture.vault().getVaultId(), written.getVaultId());+ return call.callRealMethod();+ }).when(fixture.jobs).put(anyString(), any());+ BackupJob job = fixture.startRunning();+ JsonNode stored = fixture.mapper.valueToTree(fixture.jobs.get(job.getBackupJobId()).orElseThrow());+ assertEquals(fixture.vault().getVaultId(), stored.path("FlociVaultId").asText(),+ "New jobs must persist the captured incarnation");+ }+ }++ @Test+ void capturedIncarnationRejectsNewerSameArnReplacement() throws Exception {+ assertReplacementRejected(200L);+ }++ @Test+ void capturedIncarnationRequiresEqualityEvenForOlderSameArnReplacement() throws Exception {+ assertReplacementRejected(99L);+ }++ private void assertReplacementRejected(long replacementDate) throws Exception {+ try (Fixture fixture = new Fixture()) {+ fixture.vault().setCreationDate(100L);+ BackupJob job = fixture.startRunning();+ String originalArn = fixture.vault().getBackupVaultArn();+ fixture.service.deleteBackupVault(VAULT, REGION);+ BackupVault replacement = fixture.service.createBackupVault(VAULT, null, null, Map.of(), REGION);+ replacement.setCreationDate(replacementDate);+ assertEquals(originalArn, replacement.getBackupVaultArn());+ assertNotEquals(job.getVaultId(), replacement.getVaultId());+ fixture.complete(job);+ assertEquals("FAILED", fixture.service.describeBackupJob(job.getBackupJobId()).getState());+ assertTrue(fixture.points.keys().isEmpty());+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());+ fixture.service.deleteBackupVault(VAULT, REGION);+ }+ }++ @Test+ void legacyAbsentIdentityAllowsOlderAndSameSecondVaults() throws Exception {+ for (long vaultDate : new long[]{99L, 100L}) {+ try (Fixture fixture = new Fixture()) {+ fixture.vault().setCreationDate(vaultDate);+ BackupJob legacy = fixture.legacy("RUNNING", 100L);+ fixture.complete(legacy);+ assertEquals("COMPLETED", fixture.service.describeBackupJob(legacy.getBackupJobId()).getState());+ assertEquals(1, fixture.vault().getNumberOfRecoveryPoints());+ assertEquals("arn:aws:backup:" + REGION + ":" + ACCOUNT + ":recovery-point:legacy",+ legacy.getRecoveryPointArn());+ assertEquals(ACCOUNT, legacy.getAccountId());+ assertNull(legacy.getVaultId(), "Recovery must not invent a missing captured id");+ }+ }+ }++ @Test+ void legacyAbsentIdentityRejectsVaultNewerThanJob() throws Exception {+ try (Fixture fixture = new Fixture()) {+ fixture.vault().setCreationDate(101L);+ BackupJob legacy = fixture.legacy("RUNNING", 100L);+ fixture.complete(legacy);+ assertEquals("FAILED", fixture.service.describeBackupJob(legacy.getBackupJobId()).getState());+ assertTrue(fixture.points.keys().isEmpty());+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());+ }+ }++ @Test+ void restartKeepsLegacyStartAndCompletionDeadlinesInOriginalRegion() {+ try (Fixture fixture = new Fixture()) {+ long now = Instant.now().getEpochSecond();+ fixture.vault().setCreationDate(now - 20);+ BackupJob expired = fixture.legacy("CREATED", now - 10);+ expired.setStartBy(now - 1);+ BackupJob future = fixture.legacy("RUNNING", now - 10);+ future.setBackupJobId("future");+ future.setExpectedCompletionDate(now + 3600);+ fixture.jobs.put("future", future);+ fixture.jobs.put("legacy", expired);+ fixture.service.resumeJobs();+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ assertEquals("EXPIRED", fixture.service.describeBackupJob("legacy").getState()));+ assertEquals("RUNNING", fixture.service.describeBackupJob("future").getState());+ assertEquals(now + 3600, fixture.service.describeBackupJob("future").getExpectedCompletionDate());+ assertTrue(fixture.points.keys().isEmpty());+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());+ }+ }++ @Test+ void sameSecondSameNameSameArnReplacementRejectsCapturedIdWithoutPointsOrNotifications() throws Exception {+ try (Fixture fixture = new Fixture()) {+ BackupJob job = fixture.startRunning();+ BackupVault original = fixture.vault();+ fixture.service.deleteBackupVault(VAULT, REGION);+ BackupVault replacement = fixture.service.createBackupVault(VAULT, null, null, Map.of(), REGION);+ replacement.setCreationDate(original.getCreationDate());+ fixture.service.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_COMPLETED"));+ assertEquals(original.getBackupVaultName(), replacement.getBackupVaultName());+ assertEquals(original.getBackupVaultArn(), replacement.getBackupVaultArn());+ assertEquals(original.getCreationDate(), replacement.getCreationDate());+ assertNotEquals(original.getVaultId(), replacement.getVaultId());+ fixture.complete(job);+ assertEquals("FAILED", fixture.service.describeBackupJob(job.getBackupJobId()).getState());+ assertTrue(fixture.points.keys().isEmpty());+ assertEquals(0, fixture.vault().getNumberOfRecoveryPoints());+ verifyNoInteractions(fixture.sns);+ }+ }++ @Test+ void jsonPersistenceRoundtripAndServiceRecreationAcceptOriginalAndRejectReplacement() throws Exception {+ for (boolean replace : List.of(false, true)) {+ try (Fixture fixture = new Fixture()) {+ BackupJob job = fixture.startRunning();+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);+ BackupVault original = fixture.vault();+ String capturedId = original.getVaultId();+ String jobJson = fixture.mapper.writeValueAsString(job);+ String vaultJson = fixture.mapper.writeValueAsString(original);+ fixture.service.shutdown();+ fixture.jobs.put(job.getBackupJobId(), fixture.mapper.readValue(jobJson, BackupJob.class));+ fixture.vaults.put(REGION + ":" + VAULT, fixture.mapper.readValue(vaultJson, BackupVault.class));+ fixture.service = fixture.newService();+ assertEquals(capturedId, fixture.service.describeBackupJob(job.getBackupJobId()).getVaultId());+ assertEquals(capturedId, fixture.vault().getVaultId());+ if (replace) {+ fixture.service.deleteBackupVault(VAULT, REGION);+ fixture.service.createBackupVault(VAULT, null, null, Map.of(), REGION)+ .setCreationDate(original.getCreationDate());+ }+ fixture.service.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_COMPLETED"));+ fixture.service.resumeJobs();+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertEquals(+ replace ? "FAILED" : "COMPLETED",+ fixture.service.describeBackupJob(job.getBackupJobId()).getState()));+ assertEquals(capturedId, fixture.service.describeBackupJob(job.getBackupJobId()).getVaultId());+ assertEquals(replace ? 0 : 1, fixture.points.keys().size());+ assertEquals(replace ? 0 : 1, fixture.vault().getNumberOfRecoveryPoints());+ if (replace) {+ verifyNoInteractions(fixture.sns);+ } else {+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ verify(fixture.sns).publish(eq(TOPIC), isNull(), isNull(), anyString(),+ eq("Notification from AWS Backup"), anyMap(), eq(REGION)));+ }+ }+ }+ }++ @Test+ void resumedCompletionPreservesSnsPayloadAndAttributesAndFiltersOtherEvents() throws Exception {+ try (Fixture fixture = new Fixture()) {+ fixture.service.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_FAILED"));+ BackupJob quiet = fixture.startRunning();+ quiet.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);+ fixture.service.shutdown();+ fixture.service = fixture.newService();+ fixture.service.resumeJobs();+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ assertEquals("COMPLETED", fixture.service.describeBackupJob(quiet.getBackupJobId()).getState()));+ // The terminal state is visible before notify returns; wait for the completion monitor+ // before changing the event filter for the next job.+ synchronized (fixture.vaultMonitor()) {+ verifyNoInteractions(fixture.sns);+ fixture.service.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_COMPLETED"));+ }+ BackupJob job = fixture.startRunning();+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);+ fixture.service.shutdown();+ fixture.service = fixture.newService();+ fixture.service.resumeJobs();+ @SuppressWarnings("unchecked")+ ArgumentCaptor<Map<String, MessageAttributeValue>> attrs = ArgumentCaptor.forClass(Map.class);+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ verify(fixture.sns).publish(eq(TOPIC), isNull(), isNull(), eq(+ "An AWS Backup job was completed successfully. Recovery point ARN: "+ + job.getRecoveryPointArn() + ". Resource ARN : " + job.getResourceArn()+ + ". BackupJob ID : " + job.getBackupJobId()),+ eq("Notification from AWS Backup"), attrs.capture(), eq(REGION)));+ Map<String, MessageAttributeValue> expected = Map.of(+ "EventType", new MessageAttributeValue("BACKUP_JOB", "String"),+ "State", new MessageAttributeValue("COMPLETED", "String"),+ "AccountId", new MessageAttributeValue(ACCOUNT, "String"),+ "Id", new MessageAttributeValue(job.getBackupJobId(), "String"),+ "StartTime", new MessageAttributeValue(Instant.ofEpochSecond(job.getCreationDate()).toString(), "String"));+ assertEquals(fixture.mapper.valueToTree(expected), fixture.mapper.valueToTree(attrs.getValue()));+ fixture.service.resumeJobs();+ fixture.complete(job);+ verifyNoMoreInteractions(fixture.sns);+ }+ }++ @Test+ void resumedPublishFailureDoesNotUndoCompletionOrPoint() {+ try (Fixture fixture = new Fixture()) {+ fixture.service.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_COMPLETED"));+ doThrow(new IllegalStateException("offline topic")).when(fixture.sns)+ .publish(anyString(), isNull(), isNull(), anyString(), anyString(), anyMap(), anyString());+ BackupJob job = fixture.startRunning();+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);+ fixture.service.shutdown();+ fixture.service = fixture.newService();+ fixture.service.resumeJobs();+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> {+ verify(fixture.sns).publish(eq(TOPIC), isNull(), isNull(), anyString(), anyString(), anyMap(), eq(REGION));+ assertEquals("COMPLETED", fixture.service.describeBackupJob(job.getBackupJobId()).getState());+ assertEquals(1, fixture.points.keys().size());+ assertEquals(1, fixture.vault().getNumberOfRecoveryPoints());+ });+ }+ }++ @Test+ void terminalJobsCannotBeStoppedOrCompletedAgain() throws Exception {+ for (String state : List.of("COMPLETED", "ABORTED", "FAILED", "EXPIRED", "PARTIAL")) {+ try (Fixture fixture = new Fixture()) {+ BackupJob job = fixture.legacy(state, Instant.now().getEpochSecond());+ AwsException error = assertThrows(AwsException.class, () -> fixture.service.stopBackupJob("legacy"));+ assertEquals("InvalidRequestException", error.getErrorCode());+ fixture.complete(job);+ fixture.service.resumeJobs();+ assertEquals(state, fixture.service.describeBackupJob("legacy").getState());+ assertNull(job.getRecoveryPointArn());+ assertTrue(fixture.points.keys().isEmpty());+ verifyNoInteractions(fixture.sns);+ }+ }+ }++ @Test+ void persistentJsonStoresReloadCapturedIdentityBeforeResumingOriginalOrReplacement(@TempDir Path directory)+ throws Exception {+ for (boolean replace : List.of(false, true)) {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ Path path = directory.resolve(Boolean.toString(replace));+ when(config.storage().persistentPath()).thenReturn(path.toString());+ when(config.defaultAccountId()).thenReturn(ACCOUNT);+ when(config.services().backup().jobCompletionDelaySeconds()).thenReturn(3600);+ ServiceConfigAccess access = mock(ServiceConfigAccess.class);+ when(access.storageMode("backup")).thenReturn("persistent");+ StorageFactory firstStorage = new StorageFactory(config, access);+ RegionResolver resolver = new RegionResolver("us-east-1", ACCOUNT);+ ObjectMapper mapper = new ObjectMapper();+ SnsService sns = mock(SnsService.class);+ BackupService first = new BackupService(firstStorage, config, resolver, mapper, sns);+ String jobId;+ String capturedId;+ long capturedDate;+ try {+ BackupVault vault = first.createBackupVault(VAULT, null, null, Map.of(), REGION);+ capturedId = vault.getVaultId();+ capturedDate = vault.getCreationDate();+ BackupJob job = first.startBackupJob(VAULT, "arn:aws:s3:::persisted", "role", null, REGION);+ jobId = job.getBackupJobId();+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);+ AccountAwareStorageBackend<BackupJob> jobs = firstStorage.create("backup", "backup-jobs.json",+ new TypeReference<Map<String, BackupJob>>() {});+ jobs.put(jobId, job);+ } finally {+ first.shutdown();+ firstStorage.shutdownAll();+ }+ JsonNode disk = mapper.readTree(path.resolve("backup-jobs.json").toFile()).path(ACCOUNT + "/" + jobId);+ assertEquals(capturedId, disk.path("FlociVaultId").asText());+ StorageFactory reloadedStorage = new StorageFactory(config, access);+ BackupService restarted = new BackupService(reloadedStorage, config, resolver, mapper, sns);+ try {+ assertEquals(capturedId, restarted.describeBackupJob(jobId).getVaultId());+ assertEquals(capturedId, restarted.describeBackupVault(VAULT, REGION).getVaultId());+ if (replace) {+ restarted.deleteBackupVault(VAULT, REGION);+ BackupVault replacement = restarted.createBackupVault(VAULT, null, null, Map.of(), REGION);+ replacement.setCreationDate(capturedDate);+ assertNotEquals(capturedId, replacement.getVaultId());+ }+ restarted.putBackupVaultNotifications(VAULT, REGION, TOPIC, List.of("BACKUP_JOB_COMPLETED"));+ restarted.resumeJobs();+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertEquals(+ replace ? "FAILED" : "COMPLETED", restarted.describeBackupJob(jobId).getState()));+ assertEquals(replace ? 0 : 1, restarted.listRecoveryPointsByBackupVault(VAULT, REGION).size());+ assertEquals(replace ? 0 : 1, restarted.describeBackupVault(VAULT, REGION).getNumberOfRecoveryPoints());+ assertEquals(capturedId, restarted.describeBackupJob(jobId).getVaultId());+ if (replace) {+ verifyNoInteractions(sns);+ } else {+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ verify(sns).publish(eq(TOPIC), isNull(), isNull(), anyString(), anyString(), anyMap(), eq(REGION)));+ }+ } finally {+ restarted.shutdown();+ reloadedStorage.shutdownAll();+ }+ }+ }++ @Test+ void capturedIdStillRequiresOriginalArnAfterJsonRoundtrip() throws Exception {+ for (boolean matchingArn : List.of(true, false)) {+ try (Fixture fixture = new Fixture()) {+ BackupJob job = fixture.startRunning();+ String capturedId = job.getVaultId();+ job = fixture.mapper.readValue(fixture.mapper.writeValueAsString(job), BackupJob.class);+ if (!matchingArn) {+ job.setBackupVaultArn(job.getBackupVaultArn() + "-different");+ }+ fixture.jobs.put(job.getBackupJobId(), job);+ fixture.complete(job);+ assertEquals(matchingArn ? "COMPLETED" : "FAILED", job.getState());+ assertEquals(matchingArn ? 1 : 0, fixture.points.keys().size());+ assertEquals(matchingArn ? 1 : 0, fixture.vault().getNumberOfRecoveryPoints());+ assertEquals(capturedId, job.getVaultId());+ assertEquals(fixture.vault().getVaultId(), job.getVaultId());+ }+ }+ }++ @Test+ void legacyMissingCompletionDeadlineUsesCreationPlusConfiguredDelay() {+ try (Fixture fixture = new Fixture()) {+ long now = Instant.now().getEpochSecond();+ fixture.vault().setCreationDate(now - 7200);+ BackupJob job = fixture.legacy("CREATED", now - 3601);+ job.setExpectedCompletionDate(null);+ fixture.service.resumeJobs();+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ assertEquals("COMPLETED", fixture.service.describeBackupJob("legacy").getState()));+ assertEquals(1, fixture.points.keys().size());+ assertEquals(1, fixture.vault().getNumberOfRecoveryPoints());+ assertTrue(job.getRecoveryPointArn().contains(":" + REGION + ":" + ACCOUNT + ":"));+ }+ }++ private static class Fixture implements AutoCloseable {+ private final Map<String, AccountAwareStorageBackend<?>> stores = new HashMap<>();+ private final StorageFactory storage = mock(StorageFactory.class);+ private final EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ private final RegionResolver resolver = new RegionResolver("us-east-1", ACCOUNT);+ private final ObjectMapper mapper = new ObjectMapper();+ private final SnsService sns = mock(SnsService.class);+ private final AccountAwareStorageBackend<BackupVault> vaults;+ private final AccountAwareStorageBackend<BackupJob> jobs;+ private final AccountAwareStorageBackend<RecoveryPoint> points;+ private BackupService service;++ Fixture() {+ when(storage.create(eq("backup"), anyString(), any())).thenAnswer(call ->+ stores.computeIfAbsent(call.getArgument(1), ignored ->+ spy(AccountAwareStorageBackend.inMemory(ACCOUNT))));+ when(config.services().backup().jobCompletionDelaySeconds()).thenReturn(3600);+ service = newService();+ vaults = store("backup-vaults.json");+ jobs = store("backup-jobs.json");+ points = store("backup-recovery-points.json");+ service.createBackupVault(VAULT, null, null, Map.of(), REGION);+ }++ @SuppressWarnings("unchecked")+ private <T> AccountAwareStorageBackend<T> store(String name) {+ return (AccountAwareStorageBackend<T>) stores.get(name);+ }++ private BackupService newService() {+ return new BackupService(storage, config, resolver, mapper, sns);+ }++ private BackupVault vault() {+ return service.describeBackupVault(VAULT, REGION);+ }++ private BackupJob startRunning() {+ BackupJob job = service.startBackupJob(VAULT, "arn:aws:s3:::review-bucket", "role", null, REGION);+ job.setState("RUNNING");+ jobs.put(job.getBackupJobId(), job);+ return job;+ }++ private BackupJob legacy(String state, long creationDate) {+ BackupJob job = new BackupJob();+ job.setBackupJobId("legacy");+ job.setBackupVaultName(VAULT);+ job.setBackupVaultArn(vault().getBackupVaultArn());+ job.setResourceArn("arn:aws:s3:::review-bucket");+ job.setAccountId(ACCOUNT);+ job.setState(state);+ job.setCreationDate(creationDate);+ job.setStartBy(Instant.now().getEpochSecond() + 3600);+ job.setExpectedCompletionDate(Instant.now().getEpochSecond() - 1);+ jobs.put(job.getBackupJobId(), job);+ return job;+ }++ private void complete(BackupJob job) throws Exception {+ Method method = BackupService.class.getDeclaredMethod("completeJob", String.class, String.class, String.class);+ method.setAccessible(true);+ method.invoke(service, job.getBackupJobId(), VAULT, REGION);+ }++ private Object vaultMonitor() throws Exception {+ Field field = BackupService.class.getDeclaredField("vaultLocks");+ field.setAccessible(true);+ Map<?, ?> locks = (Map<?, ?>) field.get(service);+ return locks.entrySet().stream()+ .filter(entry -> entry.getKey().toString().endsWith(REGION + ":" + VAULT))+ .map(Map.Entry::getValue).findFirst().orElseThrow();+ }++ @Override+ public void close() {+ service.shutdown();+ }+ }+}