Light, fluffy, and always free - The AWS Local Emulator alternative
fix(backup): publish BACKUP_JOB_COMPLETED to the vault's SNS topic (#5623)
Allen Sanbornc23641457a9f
diff --git a/docs/services/backup.md b/docs/services/backup.mdindex 2ed4df039d..29549ba0ac 100644--- a/docs/services/backup.md+++ b/docs/services/backup.md@@ -202,11 +202,16 @@ Actual backup is simulated — no data is read from or written to the referenced## Not Yet Supported-- **Notification delivery.** `PutBackupVaultNotifications` stores the configuration and- `GetBackupVaultNotifications` reports it, but no vault event is ever published to the SNS- topic: a subscriber receives nothing when a backup job completes. The configuration is- modelled, the delivery is not, so a Terraform configuration that attaches notifications- applies and reads back correctly while a test asserting on a received message will fail.+- **Notification delivery, beyond `BACKUP_JOB_COMPLETED`.** When a backup job completes in a+ vault whose `BackupVaultEvents` include `BACKUP_JOB_COMPLETED`, Floci publishes one message to+ the configured `SNSTopicArn` with AWS's documented subject (`Notification from AWS Backup`), body+ (recovery point, resource and job ID) and message attributes (`EventType`, `State`, `AccountId`,+ `Id`, `StartTime`), so subscription filter policies on those attributes match. That is the only+ event delivered. Every other event --+ including `BACKUP_JOB_STARTED`, `BACKUP_JOB_SUCCESSFUL`, `BACKUP_JOB_FAILED` and all+ `RESTORE_JOB_*` and `COPY_JOB_*` events -- is stored and reported by+ `GetBackupVaultNotifications` but never published. A failed publish (for example, a topic that does not+ exist) is logged and dropped without affecting the job.- **Access policy enforcement.** `PutBackupVaultAccessPolicy` stores a resource policy and`GetBackupVaultAccessPolicy` reports it, but no authorization path consults it: a policydenying a vault operation does not prevent that operation, even with IAM enforcementdiff --git a/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java b/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.javaindex fa0ab49a35..de87ee7cbb 100644--- a/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java+++ b/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java@@ -65,7 +65,7 @@ public class BackupController {public Response describeBackupVault(@Context HttpHeaders headers,@PathParam("backupVaultName") String vaultName) {String region = regionResolver.resolveRegion(headers);- return Response.ok(service.describeBackupVault(vaultName, region)).build();+ return Response.ok(vaultJson(service.describeBackupVault(vaultName, region))).build();}@DELETE@@ -84,10 +84,17 @@ public class BackupController {List<BackupVault> vaults = service.listBackupVaults(region);ObjectNode out = objectMapper.createObjectNode();ArrayNode list = out.putArray("BackupVaultList");- vaults.forEach(list::addPOJO);+ vaults.forEach(v -> list.add(vaultJson(v)));return Response.ok(out).build();}+ /** The vault as AWS reports it: its stored form minus Floci's internal incarnation id. */+ private ObjectNode vaultJson(BackupVault vault) {+ ObjectNode node = objectMapper.valueToTree(vault);+ node.remove(BackupVault.VAULT_ID_FIELD);+ return node;+ }+// ── Vault access policy ────────────────────────────────────────────────────//// These three used to be one unconditional GET that always answereddiff --git a/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java b/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.javaindex 0c49935fb8..8729ab2fc1 100644--- a/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java+++ b/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java@@ -11,12 +11,15 @@ import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.backup.model.*;+import io.github.hectorvent.floci.services.sns.SnsService;+import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.time.Instant;+import java.time.format.DateTimeFormatter;import java.time.temporal.ChronoUnit;import java.util.*;import java.util.concurrent.ConcurrentHashMap;@@ -43,6 +46,7 @@ public class BackupService {private final StorageBackend<String, BackupVaultNotifications> notificationStore;private final RegionResolver regionResolver;+ private final SnsService snsService;private final int jobCompletionDelaySeconds;private final ObjectMapper objectMapper;@@ -63,7 +67,7 @@ public class BackupService {@Injectpublic BackupService(StorageFactory storageFactory, EmulatorConfig config, RegionResolver regionResolver,- ObjectMapper objectMapper) {+ ObjectMapper objectMapper, SnsService snsService) {this.vaultStore = storageFactory.create("backup", "backup-vaults.json", new TypeReference<>() {});this.planStore = storageFactory.create("backup", "backup-plans.json", new TypeReference<>() {});this.selectionStore = storageFactory.create("backup", "backup-selections.json", new TypeReference<>() {});@@ -73,6 +77,7 @@ public class BackupService {this.notificationStore = storageFactory.create("backup", "backup-vault-notifications.json", new TypeReference<>() {});this.regionResolver = regionResolver;this.objectMapper = objectMapper;+ this.snsService = snsService;this.jobCompletionDelaySeconds = config.services().backup().jobCompletionDelaySeconds();}@@ -108,6 +113,7 @@ public class BackupService {vault.setBackupVaultArn(regionResolver.buildArn("backup", region, "backup-vault:" + vaultName));vault.setEncryptionKeyArn(encryptionKeyArn);vault.setCreationDate(Instant.now().getEpochSecond());+ vault.setVaultId(UUID.randomUUID().toString());vault.setCreatorRequestId(creatorRequestId);vault.setNumberOfRecoveryPoints(0);vault.setTags(tags);@@ -150,7 +156,7 @@ public class BackupService {// Dependents first, the vault record last. There is no transaction across three// stores, so whatever sits between the first delete and the last is observable, and// this order keeps the observable state a vault with no configuration rather than a- // configuration with no vault. The sub-resource keys carry this vault's creation date,+ // configuration with no vault. The sub-resource keys carry this vault's FlociVaultId,// so these two deletes cannot reach a configuration belonging to a later vault that// has taken the same name -- which is what made the previous order lose data.accessPolicyStore.delete(subResourceKey(region, vault));@@ -601,13 +607,16 @@ public class BackupService {String accountId = regionResolver.getAccountId();job.setAccountId(accountId);jobStore.put(jobId, job);+ // Pinned now, not looked up by name at completion: if this vault is deleted and the name+ // reused while the job runs, the job must not notify the replacement vault's topic.+ String notificationKey = subResourceKey(region, vault);// The scheduler thread has no request context; run in the caller's account// so the account-aware stores and buildArn resolve the right partition.scheduler.schedule(() -> RequestScopes.runAs(accountId,() -> transitionJob(jobId, vaultName, region)), 1, TimeUnit.SECONDS);scheduler.schedule(() -> RequestScopes.runAs(accountId,- () -> completeJob(jobId, vaultName, region)), jobCompletionDelaySeconds, TimeUnit.SECONDS);+ () -> completeJob(jobId, vaultName, region, notificationKey)), jobCompletionDelaySeconds, TimeUnit.SECONDS);return job;}@@ -725,7 +734,7 @@ public class BackupService {});}- private void completeJob(String jobId, String vaultName, String region) {+ private void completeJob(String jobId, String vaultName, String region, String notificationKey) {jobStore.get(jobId).ifPresent(job -> {if ("RUNNING".equals(job.getState())) {long now = Instant.now().getEpochSecond();@@ -757,10 +766,45 @@ public class BackupService {incrementVaultCount(vaultName, region);LOG.infov("Backup job {0} completed, recovery point: {1}", jobId, rpArn);+ notifyJobCompleted(job, vaultName, region, notificationKey);}});}+ /**+ * Publishes BACKUP_JOB_COMPLETED to the vault's SNS topic, if the vault asked for it.+ * Runs after the job and recovery point are stored, and never fails the completion: AWS+ * does not undo a backup because its notification could not be delivered.+ *+ * <p>{@code notificationKey} is the sub-resource key of the vault the job started in. A vault+ * deleted mid-job takes its configuration with it, so the job publishes nothing rather than+ * reaching a later vault that took the same name.+ */+ private void notifyJobCompleted(BackupJob job, String vaultName, String region, String notificationKey) {+ try {+ notificationStore.get(notificationKey)+ .filter(n -> n.getBackupVaultEvents() != null+ && n.getBackupVaultEvents().contains("BACKUP_JOB_COMPLETED"))+ .ifPresent(n -> snsService.publish(n.getSnsTopicArn(), null, null,+ "An AWS Backup job was completed successfully. Recovery point ARN: "+ + job.getRecoveryPointArn() + ". Resource ARN : " + job.getResourceArn()+ + ". BackupJob ID : " + job.getBackupJobId(),+ "Notification from AWS Backup",+ // AWS's documented "Backup job completed" example; subscription+ // filter policies match on these.+ Map.of("EventType", new MessageAttributeValue("BACKUP_JOB", "String"),+ "State", new MessageAttributeValue("COMPLETED", "String"),+ "AccountId", new MessageAttributeValue(job.getAccountId(), "String"),+ "Id", new MessageAttributeValue(job.getBackupJobId(), "String"),+ "StartTime", new MessageAttributeValue(DateTimeFormatter.ISO_INSTANT+ .format(Instant.ofEpochSecond(job.getCreationDate())), "String")),+ AwsArnUtils.regionOrDefault(n.getSnsTopicArn(), region)));+ } catch (Exception e) {+ LOG.warnv(e, "Failed to publish BACKUP_JOB_COMPLETED for job {0} in vault {1}",+ job.getBackupJobId(), vaultName);+ }+ }+private void abortJob(String jobId) {jobStore.get(jobId).ifPresent(job -> {if ("ABORTING".equals(job.getState())) {@@ -875,20 +919,19 @@ public class BackupService {* the second worse, because two creates racing the existence check would both sweep, and the* later sweep would delete a configuration applied against the earlier vault.*- * <p>Including the creation date removes the shared key instead of policing it. A new vault- * cannot read, and cleanup for an old vault cannot touch, a record belonging to a different- * incarnation of the name -- no sweep, and no dependence on the order two stores are written- * in. Deleting a vault still removes its own records, now because they are its own rather- * than because nothing else has claimed them yet.- *- * <p>The creation date is only unique per name because {@code createBackupVault} holds a- * per-name monitor across its check and its write, so two vaults of one name cannot exist- * and cannot be created in the same second. Without that lock this key would still collide- * and the two vaults could read each other's configuration. The delete path also removes its- * own sub-resources, which covers a record surviving for any other reason.+ * <p>Including the vault's {@code FlociVaultId}, a random id minted at creation, removes the+ * shared key instead of policing it. A new vault cannot read, and cleanup for an old vault+ * cannot touch, a record belonging to a different incarnation of the name -- no sweep, and+ * no dependence on the order two stores are written in, and no dependence on the clock: a+ * vault deleted and recreated within the same second still gets a different key. Deleting a+ * vault still removes its own records, now because they are its own rather than because+ * nothing else has claimed them yet.*/private static String subResourceKey(String region, BackupVault vault) {- return vaultKey(region, vault.getBackupVaultName()) + ":" + vault.getCreationDate();+ // Vaults stored before FlociVaultId existed keep their creation-date key, so their+ // stored policy and notification configuration are still found.+ String incarnation = vault.getVaultId() != null ? vault.getVaultId() : String.valueOf(vault.getCreationDate());+ return vaultKey(region, vault.getBackupVaultName()) + ":" + incarnation;}private static String vaultKey(BackupVault vault) {diff --git a/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupVault.java b/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupVault.javaindex 83ac954f09..b637315aa6 100644--- a/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupVault.java+++ b/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupVault.java@@ -55,6 +55,16 @@ public class BackupVault {@JsonProperty("MaxRetentionDays")private Long maxRetentionDays;+ // Floci-internal, not an AWS member: identifies this incarnation of the vault name, so+ // sub-resources and in-flight jobs bind to the vault rather than to a name that a+ // delete-and-recreate can hand to a different vault within the same second. Persisted,+ // but BackupController strips it from every response. Null on vaults stored before it+ // existed; BackupService falls back to the creation date for those.+ @JsonProperty(VAULT_ID_FIELD)+ private String vaultId;++ public static final String VAULT_ID_FIELD = "FlociVaultId";+public BackupVault() {}public String getBackupVaultName() { return backupVaultName; }@@ -63,6 +73,9 @@ public class BackupVault {public String getBackupVaultArn() { return backupVaultArn; }public void setBackupVaultArn(String backupVaultArn) { this.backupVaultArn = backupVaultArn; }+ public String getVaultId() { return vaultId; }+ public void setVaultId(String vaultId) { this.vaultId = vaultId; }+public String getEncryptionKeyArn() { return encryptionKeyArn; }public void setEncryptionKeyArn(String encryptionKeyArn) { this.encryptionKeyArn = encryptionKeyArn; }diff --git a/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.javaindex d02b9fe389..caa764f3b0 100644--- a/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java@@ -1,17 +1,28 @@package io.github.hectorvent.floci.services.backup;import io.quarkus.test.junit.QuarkusTest;+import io.restassured.path.json.JsonPath;+import io.restassured.specification.RequestSpecification;import org.junit.jupiter.api.MethodOrderer;import org.junit.jupiter.api.Order;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.TestMethodOrder;import java.time.Duration;+import java.time.Instant;+import java.util.List;import java.util.Map;+import java.util.concurrent.CopyOnWriteArrayList;+import java.util.logging.Handler;+import java.util.logging.Level;+import java.util.logging.LogRecord;+import java.util.logging.Logger;import static io.restassured.RestAssured.given;import static org.awaitility.Awaitility.await;+import static org.hamcrest.MatcherAssert.assertThat;import static org.hamcrest.Matchers.*;+import static org.junit.jupiter.api.Assertions.assertEquals;/*** Integration tests for AWS Backup via REST JSON protocol.@@ -1421,4 +1432,179 @@ class BackupIntegrationTest {awaitJobState(OTHER_ACCOUNT_AUTH, jid, "ABORTED");}++ // ── Vault notifications: delivery ──────────────────────────────────────────++ private static String createQueue(String name) {+ return given().contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateQueue").formParam("QueueName", name)+ .when().post("/").then().statusCode(200)+ .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl");+ }++ /** A topic with {@code queueUrl} subscribed, so what SNS delivers can be read back from SQS. */+ private static String topicDeliveringTo(String name, String queueUrl) {+ return topicDeliveringTo(name, queueUrl, null);+ }++ /** As above, with the subscription filtered by {@code filterPolicy} when it is non-null. */+ private static String topicDeliveringTo(String name, String queueUrl, String filterPolicy) {+ String topicArn = given().contentType("application/x-www-form-urlencoded")+ .formParam("Action", "CreateTopic").formParam("Name", name)+ .when().post("/").then().statusCode(200)+ .extract().xmlPath().getString("CreateTopicResponse.CreateTopicResult.TopicArn");+ RequestSpecification subscribe = given().contentType("application/x-www-form-urlencoded")+ .formParam("Action", "Subscribe").formParam("TopicArn", topicArn)+ .formParam("Protocol", "sqs").formParam("Endpoint", queueUrl);+ if (filterPolicy != null) {+ subscribe = subscribe.formParam("Attributes.entry.1.key", "FilterPolicy")+ .formParam("Attributes.entry.1.value", filterPolicy);+ }+ subscribe.when().post("/").then().statusCode(200);+ return topicArn;+ }++ private static String receiveBody(String queueUrl) {+ return given().contentType("application/x-www-form-urlencoded")+ .formParam("Action", "ReceiveMessage").formParam("QueueUrl", queueUrl)+ .formParam("MaxNumberOfMessages", "1")+ .when().post("/").then().statusCode(200)+ .extract().xmlPath().getString("ReceiveMessageResponse.ReceiveMessageResult.Message.Body");+ }++ private static String startJobInNotifiedVault(String vault, String topicArn, String event) {+ given().header("Authorization", AUTH).contentType("application/json").body("{}")+ .when().put("/backup-vaults/" + vault).then().statusCode(200);+ given().header("Authorization", AUTH).contentType("application/json")+ .body("{\"SNSTopicArn\":\"%s\",\"BackupVaultEvents\":[\"%s\"]}".formatted(topicArn, event))+ .when().put("/backup-vaults/" + vault + "/notification-configuration").then().statusCode(204);+ return given().header("Authorization", AUTH).contentType("application/json")+ .body("""+ {+ "BackupVaultName": "%s",+ "ResourceArn": "%s",+ "IamRoleArn": "%s"+ }+ """.formatted(vault, RESOURCE_ARN, IAM_ROLE))+ .when().put("/backup-jobs").then().statusCode(200).extract().path("BackupJobId");+ }++ @Test+ @Order(170)+ void aCompletedJobIsPublishedOnlyToAVaultThatAskedForBackupJobCompleted() {+ String quietQueue = createQueue("backup-notify-quiet-queue");+ String quietTopic = topicDeliveringTo("backup-notify-quiet-topic", quietQueue);+ String notifiedQueue = createQueue("backup-notify-queue");+ // Filtered on the attributes AWS's "Backup job completed" example carries: a publish+ // without them is dropped by SNS and never reaches the queue.+ String notifiedTopic = topicDeliveringTo("backup-notify-topic", notifiedQueue,+ "{\"EventType\":[\"BACKUP_JOB\"],\"State\":[\"COMPLETED\"]}");++ // The quiet vault's job is started first. Completions run on one scheduler thread in+ // start order, so by the time the notified vault's message has arrived, any publish+ // the quiet vault's completion was going to make has already been made.+ String quietJob = startJobInNotifiedVault("notify-quiet-vault", quietTopic, "BACKUP_JOB_FAILED");+ String jid = startJobInNotifiedVault("notify-vault", notifiedTopic, "BACKUP_JOB_COMPLETED");++ String[] body = new String[1];+ await().atMost(Duration.ofSeconds(10)).pollInterval(Duration.ofMillis(100))+ .until(() -> (body[0] = receiveBody(notifiedQueue)) != null && !body[0].isEmpty());++ awaitJobCompleted(jid);+ String rpArn = given().header("Authorization", AUTH)+ .when().get("/backup-jobs/" + jid).then().statusCode(200)+ .extract().path("RecoveryPointArn");++ JsonPath envelope = new JsonPath(body[0]);+ assertEquals(notifiedTopic, envelope.getString("TopicArn"));+ assertEquals("Notification from AWS Backup", envelope.getString("Subject"));+ // AWS's documented body, which ends at the job ID and does not name the vault.+ assertEquals("An AWS Backup job was completed successfully. Recovery point ARN: " + rpArn+ + ". Resource ARN : " + RESOURCE_ARN + ". BackupJob ID : " + jid, envelope.getString("Message"));+ long created = given().header("Authorization", AUTH)+ .when().get("/backup-jobs/" + jid).then().statusCode(200)+ .extract().jsonPath().getLong("CreationDate");+ assertEquals(Map.of(+ "EventType", Map.of("Type", "String", "Value", "BACKUP_JOB"),+ "State", Map.of("Type", "String", "Value", "COMPLETED"),+ "AccountId", Map.of("Type", "String", "Value", "000000000000"),+ "Id", Map.of("Type", "String", "Value", jid),+ "StartTime", Map.of("Type", "String", "Value", Instant.ofEpochSecond(created).toString())),+ envelope.getMap("MessageAttributes"));++ awaitJobCompleted(quietJob);+ assertThat(receiveBody(quietQueue), anyOf(nullValue(), emptyString()));+ }++ @Test+ @Order(171)+ void aFailedPublishIsLoggedWithItsStackTraceAndDoesNotUndoTheJob() {+ List<LogRecord> records = new CopyOnWriteArrayList<>();+ Handler capture = new Handler() {+ @Override public void publish(LogRecord r) { records.add(r); }+ @Override public void flush() { }+ @Override public void close() { }+ };+ Logger log = Logger.getLogger(BackupService.class.getName());+ log.addHandler(capture);+ try {+ String missingTopic = "arn:aws:sns:us-east-1:000000000000:backup-notify-no-such-topic";+ String jid = startJobInNotifiedVault("notify-broken-vault", missingTopic, "BACKUP_JOB_COMPLETED");+ awaitJobCompleted(jid);+ await().atMost(Duration.ofSeconds(10)).until(() -> records.stream()+ .anyMatch(r -> r.getLevel().intValue() >= Level.WARNING.intValue()+ && String.valueOf(r.getMessage()).contains("BACKUP_JOB_COMPLETED")));+ assertThat("the warning carries the exception, not just its message",+ records.stream().filter(r -> String.valueOf(r.getMessage()).contains("BACKUP_JOB_COMPLETED"))+ .allMatch(r -> r.getThrown() != null));+ } finally {+ log.removeHandler(capture);+ }+ }++ @Test+ @Order(172)+ void aJobWhoseVaultWasReplacedDoesNotNotifyTheReplacement() {+ String replacementQueue = createQueue("backup-notify-replacement-queue");+ String replacementTopic = topicDeliveringTo("backup-notify-replacement-topic", replacementQueue);+ String sentinelQueue = createQueue("backup-notify-sentinel-queue");+ String sentinelTopic = topicDeliveringTo("backup-notify-sentinel-topic", sentinelQueue);+ String vault = "notify-replaced-vault";++ // No waiting for the clock: the original and its replacement are normally created in+ // the same second, and must still be told apart.+ given().header("Authorization", AUTH).contentType("application/json").body("{}")+ .when().put("/backup-vaults/" + vault).then().statusCode(200);+ // The incarnation id is Floci-internal and must not leak into the AWS response.+ given().header("Authorization", AUTH)+ .when().get("/backup-vaults/" + vault).then().statusCode(200)+ .body("$", not(hasKey("FlociVaultId")));+ given().header("Authorization", AUTH)+ .when().get("/backup-vaults/").then().statusCode(200)+ .body(not(containsString("FlociVaultId")));++ String oldJob = given().header("Authorization", AUTH).contentType("application/json")+ .body("""+ {"BackupVaultName": "%s", "ResourceArn": "%s", "IamRoleArn": "%s"}+ """.formatted(vault, RESOURCE_ARN, IAM_ROLE))+ .when().put("/backup-jobs").then().statusCode(200).extract().path("BackupJobId");+ given().header("Authorization", AUTH)+ .when().delete("/backup-vaults/" + vault).then().statusCode(204);+ // Recreates the name and points it at a topic the old job must never reach.+ startJobInNotifiedVault(vault, replacementTopic, "BACKUP_JOB_COMPLETED");+ // Completions run in start order on one thread: once this one is delivered, the+ // old job's completion, and any publish it was going to make, has already happened.+ startJobInNotifiedVault("notify-sentinel-vault", sentinelTopic, "BACKUP_JOB_COMPLETED");++ await().atMost(Duration.ofSeconds(10)).pollInterval(Duration.ofMillis(100))+ .until(() -> { String b = receiveBody(sentinelQueue); return b != null && !b.isEmpty(); });+ awaitJobCompleted(oldJob);++ // The replacement's own job does publish; the old job must not have published too.+ String first = receiveBody(replacementQueue);+ assertThat(first, not(containsString("BackupJob ID : " + oldJob)));+ String second = receiveBody(replacementQueue);+ assertThat(second == null ? "" : second, not(containsString("BackupJob ID : " + oldJob)));+ }+}