Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(backup): publish BACKUP_JOB_COMPLETED to the vault's SNS topic (#5623)

Allen Sanbornc23641457a9f
diff --git a/docs/services/backup.md b/docs/services/backup.md
index 2ed4df039d..29549ba0ac 100644
--- a/docs/services/backup.md
+++ b/docs/services/backup.md
@@ -202,11 +202,16 @@ Actual backup is simulated — no data is read from or written to the referenced
## Not Yet Supported
-- **Notification delivery.** `PutBackupVaultNotifications` stores the configuration and
- `GetBackupVaultNotifications` reports it, but no vault event is ever published to the SNS
- topic: a subscriber receives nothing when a backup job completes. The configuration is
- modelled, the delivery is not, so a Terraform configuration that attaches notifications
- applies and reads back correctly while a test asserting on a received message will fail.
+- **Notification delivery, beyond `BACKUP_JOB_COMPLETED`.** When a backup job completes in a
+ vault whose `BackupVaultEvents` include `BACKUP_JOB_COMPLETED`, Floci publishes one message to
+ the configured `SNSTopicArn` with AWS's documented subject (`Notification from AWS Backup`), body
+ (recovery point, resource and job ID) and message attributes (`EventType`, `State`, `AccountId`,
+ `Id`, `StartTime`), so subscription filter policies on those attributes match. That is the only
+ event delivered. Every other event --
+ including `BACKUP_JOB_STARTED`, `BACKUP_JOB_SUCCESSFUL`, `BACKUP_JOB_FAILED` and all
+ `RESTORE_JOB_*` and `COPY_JOB_*` events -- is stored and reported by
+ `GetBackupVaultNotifications` but never published. A failed publish (for example, a topic that does not
+ exist) is logged and dropped without affecting the job.
- **Access policy enforcement.** `PutBackupVaultAccessPolicy` stores a resource policy and
`GetBackupVaultAccessPolicy` reports it, but no authorization path consults it: a policy
denying a vault operation does not prevent that operation, even with IAM enforcement
diff --git a/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java b/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java
index fa0ab49a35..de87ee7cbb 100644
--- a/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/backup/BackupController.java
@@ -65,7 +65,7 @@ public class BackupController {
public Response describeBackupVault(@Context HttpHeaders headers,
@PathParam("backupVaultName") String vaultName) {
String region = regionResolver.resolveRegion(headers);
- return Response.ok(service.describeBackupVault(vaultName, region)).build();
+ return Response.ok(vaultJson(service.describeBackupVault(vaultName, region))).build();
}
@DELETE
@@ -84,10 +84,17 @@ public class BackupController {
List<BackupVault> vaults = service.listBackupVaults(region);
ObjectNode out = objectMapper.createObjectNode();
ArrayNode list = out.putArray("BackupVaultList");
- vaults.forEach(list::addPOJO);
+ vaults.forEach(v -> list.add(vaultJson(v)));
return Response.ok(out).build();
}
+ /** The vault as AWS reports it: its stored form minus Floci's internal incarnation id. */
+ private ObjectNode vaultJson(BackupVault vault) {
+ ObjectNode node = objectMapper.valueToTree(vault);
+ node.remove(BackupVault.VAULT_ID_FIELD);
+ return node;
+ }
+
// ── Vault access policy ────────────────────────────────────────────────────
//
// These three used to be one unconditional GET that always answered
diff --git a/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java b/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java
index 0c49935fb8..8729ab2fc1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/backup/BackupService.java
@@ -11,12 +11,15 @@ import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.backup.model.*;
+import io.github.hectorvent.floci.services.sns.SnsService;
+import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.time.Instant;
+import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@@ -43,6 +46,7 @@ public class BackupService {
private final StorageBackend<String, BackupVaultNotifications> notificationStore;
private final RegionResolver regionResolver;
+ private final SnsService snsService;
private final int jobCompletionDelaySeconds;
private final ObjectMapper objectMapper;
@@ -63,7 +67,7 @@ public class BackupService {
@Inject
public BackupService(StorageFactory storageFactory, EmulatorConfig config, RegionResolver regionResolver,
- ObjectMapper objectMapper) {
+ ObjectMapper objectMapper, SnsService snsService) {
this.vaultStore = storageFactory.create("backup", "backup-vaults.json", new TypeReference<>() {});
this.planStore = storageFactory.create("backup", "backup-plans.json", new TypeReference<>() {});
this.selectionStore = storageFactory.create("backup", "backup-selections.json", new TypeReference<>() {});
@@ -73,6 +77,7 @@ public class BackupService {
this.notificationStore = storageFactory.create("backup", "backup-vault-notifications.json", new TypeReference<>() {});
this.regionResolver = regionResolver;
this.objectMapper = objectMapper;
+ this.snsService = snsService;
this.jobCompletionDelaySeconds = config.services().backup().jobCompletionDelaySeconds();
}
@@ -108,6 +113,7 @@ public class BackupService {
vault.setBackupVaultArn(regionResolver.buildArn("backup", region, "backup-vault:" + vaultName));
vault.setEncryptionKeyArn(encryptionKeyArn);
vault.setCreationDate(Instant.now().getEpochSecond());
+ vault.setVaultId(UUID.randomUUID().toString());
vault.setCreatorRequestId(creatorRequestId);
vault.setNumberOfRecoveryPoints(0);
vault.setTags(tags);
@@ -150,7 +156,7 @@ public class BackupService {
// Dependents first, the vault record last. There is no transaction across three
// stores, so whatever sits between the first delete and the last is observable, and
// this order keeps the observable state a vault with no configuration rather than a
- // configuration with no vault. The sub-resource keys carry this vault's creation date,
+ // configuration with no vault. The sub-resource keys carry this vault's FlociVaultId,
// so these two deletes cannot reach a configuration belonging to a later vault that
// has taken the same name -- which is what made the previous order lose data.
accessPolicyStore.delete(subResourceKey(region, vault));
@@ -601,13 +607,16 @@ public class BackupService {
String accountId = regionResolver.getAccountId();
job.setAccountId(accountId);
jobStore.put(jobId, job);
+ // Pinned now, not looked up by name at completion: if this vault is deleted and the name
+ // reused while the job runs, the job must not notify the replacement vault's topic.
+ String notificationKey = subResourceKey(region, vault);
// The scheduler thread has no request context; run in the caller's account
// so the account-aware stores and buildArn resolve the right partition.
scheduler.schedule(() -> RequestScopes.runAs(accountId,
() -> transitionJob(jobId, vaultName, region)), 1, TimeUnit.SECONDS);
scheduler.schedule(() -> RequestScopes.runAs(accountId,
- () -> completeJob(jobId, vaultName, region)), jobCompletionDelaySeconds, TimeUnit.SECONDS);
+ () -> completeJob(jobId, vaultName, region, notificationKey)), jobCompletionDelaySeconds, TimeUnit.SECONDS);
return job;
}
@@ -725,7 +734,7 @@ public class BackupService {
});
}
- private void completeJob(String jobId, String vaultName, String region) {
+ private void completeJob(String jobId, String vaultName, String region, String notificationKey) {
jobStore.get(jobId).ifPresent(job -> {
if ("RUNNING".equals(job.getState())) {
long now = Instant.now().getEpochSecond();
@@ -757,10 +766,45 @@ public class BackupService {
incrementVaultCount(vaultName, region);
LOG.infov("Backup job {0} completed, recovery point: {1}", jobId, rpArn);
+ notifyJobCompleted(job, vaultName, region, notificationKey);
}
});
}
+ /**
+ * Publishes BACKUP_JOB_COMPLETED to the vault's SNS topic, if the vault asked for it.
+ * Runs after the job and recovery point are stored, and never fails the completion: AWS
+ * does not undo a backup because its notification could not be delivered.
+ *
+ * <p>{@code notificationKey} is the sub-resource key of the vault the job started in. A vault
+ * deleted mid-job takes its configuration with it, so the job publishes nothing rather than
+ * reaching a later vault that took the same name.
+ */
+ private void notifyJobCompleted(BackupJob job, String vaultName, String region, String notificationKey) {
+ try {
+ notificationStore.get(notificationKey)
+ .filter(n -> n.getBackupVaultEvents() != null
+ && n.getBackupVaultEvents().contains("BACKUP_JOB_COMPLETED"))
+ .ifPresent(n -> snsService.publish(n.getSnsTopicArn(), null, null,
+ "An AWS Backup job was completed successfully. Recovery point ARN: "
+ + job.getRecoveryPointArn() + ". Resource ARN : " + job.getResourceArn()
+ + ". BackupJob ID : " + job.getBackupJobId(),
+ "Notification from AWS Backup",
+ // AWS's documented "Backup job completed" example; subscription
+ // filter policies match on these.
+ Map.of("EventType", new MessageAttributeValue("BACKUP_JOB", "String"),
+ "State", new MessageAttributeValue("COMPLETED", "String"),
+ "AccountId", new MessageAttributeValue(job.getAccountId(), "String"),
+ "Id", new MessageAttributeValue(job.getBackupJobId(), "String"),
+ "StartTime", new MessageAttributeValue(DateTimeFormatter.ISO_INSTANT
+ .format(Instant.ofEpochSecond(job.getCreationDate())), "String")),
+ AwsArnUtils.regionOrDefault(n.getSnsTopicArn(), region)));
+ } catch (Exception e) {
+ LOG.warnv(e, "Failed to publish BACKUP_JOB_COMPLETED for job {0} in vault {1}",
+ job.getBackupJobId(), vaultName);
+ }
+ }
+
private void abortJob(String jobId) {
jobStore.get(jobId).ifPresent(job -> {
if ("ABORTING".equals(job.getState())) {
@@ -875,20 +919,19 @@ public class BackupService {
* the second worse, because two creates racing the existence check would both sweep, and the
* later sweep would delete a configuration applied against the earlier vault.
*
- * <p>Including the creation date removes the shared key instead of policing it. A new vault
- * cannot read, and cleanup for an old vault cannot touch, a record belonging to a different
- * incarnation of the name -- no sweep, and no dependence on the order two stores are written
- * in. Deleting a vault still removes its own records, now because they are its own rather
- * than because nothing else has claimed them yet.
- *
- * <p>The creation date is only unique per name because {@code createBackupVault} holds a
- * per-name monitor across its check and its write, so two vaults of one name cannot exist
- * and cannot be created in the same second. Without that lock this key would still collide
- * and the two vaults could read each other's configuration. The delete path also removes its
- * own sub-resources, which covers a record surviving for any other reason.
+ * <p>Including the vault's {@code FlociVaultId}, a random id minted at creation, removes the
+ * shared key instead of policing it. A new vault cannot read, and cleanup for an old vault
+ * cannot touch, a record belonging to a different incarnation of the name -- no sweep, and
+ * no dependence on the order two stores are written in, and no dependence on the clock: a
+ * vault deleted and recreated within the same second still gets a different key. Deleting a
+ * vault still removes its own records, now because they are its own rather than because
+ * nothing else has claimed them yet.
*/
private static String subResourceKey(String region, BackupVault vault) {
- return vaultKey(region, vault.getBackupVaultName()) + ":" + vault.getCreationDate();
+ // Vaults stored before FlociVaultId existed keep their creation-date key, so their
+ // stored policy and notification configuration are still found.
+ String incarnation = vault.getVaultId() != null ? vault.getVaultId() : String.valueOf(vault.getCreationDate());
+ return vaultKey(region, vault.getBackupVaultName()) + ":" + incarnation;
}
private static String vaultKey(BackupVault vault) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupVault.java b/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupVault.java
index 83ac954f09..b637315aa6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupVault.java
+++ b/src/main/java/io/github/hectorvent/floci/services/backup/model/BackupVault.java
@@ -55,6 +55,16 @@ public class BackupVault {
@JsonProperty("MaxRetentionDays")
private Long maxRetentionDays;
+ // Floci-internal, not an AWS member: identifies this incarnation of the vault name, so
+ // sub-resources and in-flight jobs bind to the vault rather than to a name that a
+ // delete-and-recreate can hand to a different vault within the same second. Persisted,
+ // but BackupController strips it from every response. Null on vaults stored before it
+ // existed; BackupService falls back to the creation date for those.
+ @JsonProperty(VAULT_ID_FIELD)
+ private String vaultId;
+
+ public static final String VAULT_ID_FIELD = "FlociVaultId";
+
public BackupVault() {}
public String getBackupVaultName() { return backupVaultName; }
@@ -63,6 +73,9 @@ public class BackupVault {
public String getBackupVaultArn() { return backupVaultArn; }
public void setBackupVaultArn(String backupVaultArn) { this.backupVaultArn = backupVaultArn; }
+ public String getVaultId() { return vaultId; }
+ public void setVaultId(String vaultId) { this.vaultId = vaultId; }
+
public String getEncryptionKeyArn() { return encryptionKeyArn; }
public void setEncryptionKeyArn(String encryptionKeyArn) { this.encryptionKeyArn = encryptionKeyArn; }
diff --git a/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java
index d02b9fe389..caa764f3b0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/backup/BackupIntegrationTest.java
@@ -1,17 +1,28 @@
package io.github.hectorvent.floci.services.backup;
import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.path.json.JsonPath;
+import io.restassured.specification.RequestSpecification;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
import static io.restassured.RestAssured.given;
import static org.awaitility.Awaitility.await;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* Integration tests for AWS Backup via REST JSON protocol.
@@ -1421,4 +1432,179 @@ class BackupIntegrationTest {
awaitJobState(OTHER_ACCOUNT_AUTH, jid, "ABORTED");
}
+
+ // ── Vault notifications: delivery ──────────────────────────────────────────
+
+ private static String createQueue(String name) {
+ return given().contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "CreateQueue").formParam("QueueName", name)
+ .when().post("/").then().statusCode(200)
+ .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl");
+ }
+
+ /** A topic with {@code queueUrl} subscribed, so what SNS delivers can be read back from SQS. */
+ private static String topicDeliveringTo(String name, String queueUrl) {
+ return topicDeliveringTo(name, queueUrl, null);
+ }
+
+ /** As above, with the subscription filtered by {@code filterPolicy} when it is non-null. */
+ private static String topicDeliveringTo(String name, String queueUrl, String filterPolicy) {
+ String topicArn = given().contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "CreateTopic").formParam("Name", name)
+ .when().post("/").then().statusCode(200)
+ .extract().xmlPath().getString("CreateTopicResponse.CreateTopicResult.TopicArn");
+ RequestSpecification subscribe = given().contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "Subscribe").formParam("TopicArn", topicArn)
+ .formParam("Protocol", "sqs").formParam("Endpoint", queueUrl);
+ if (filterPolicy != null) {
+ subscribe = subscribe.formParam("Attributes.entry.1.key", "FilterPolicy")
+ .formParam("Attributes.entry.1.value", filterPolicy);
+ }
+ subscribe.when().post("/").then().statusCode(200);
+ return topicArn;
+ }
+
+ private static String receiveBody(String queueUrl) {
+ return given().contentType("application/x-www-form-urlencoded")
+ .formParam("Action", "ReceiveMessage").formParam("QueueUrl", queueUrl)
+ .formParam("MaxNumberOfMessages", "1")
+ .when().post("/").then().statusCode(200)
+ .extract().xmlPath().getString("ReceiveMessageResponse.ReceiveMessageResult.Message.Body");
+ }
+
+ private static String startJobInNotifiedVault(String vault, String topicArn, String event) {
+ given().header("Authorization", AUTH).contentType("application/json").body("{}")
+ .when().put("/backup-vaults/" + vault).then().statusCode(200);
+ given().header("Authorization", AUTH).contentType("application/json")
+ .body("{\"SNSTopicArn\":\"%s\",\"BackupVaultEvents\":[\"%s\"]}".formatted(topicArn, event))
+ .when().put("/backup-vaults/" + vault + "/notification-configuration").then().statusCode(204);
+ return given().header("Authorization", AUTH).contentType("application/json")
+ .body("""
+ {
+ "BackupVaultName": "%s",
+ "ResourceArn": "%s",
+ "IamRoleArn": "%s"
+ }
+ """.formatted(vault, RESOURCE_ARN, IAM_ROLE))
+ .when().put("/backup-jobs").then().statusCode(200).extract().path("BackupJobId");
+ }
+
+ @Test
+ @Order(170)
+ void aCompletedJobIsPublishedOnlyToAVaultThatAskedForBackupJobCompleted() {
+ String quietQueue = createQueue("backup-notify-quiet-queue");
+ String quietTopic = topicDeliveringTo("backup-notify-quiet-topic", quietQueue);
+ String notifiedQueue = createQueue("backup-notify-queue");
+ // Filtered on the attributes AWS's "Backup job completed" example carries: a publish
+ // without them is dropped by SNS and never reaches the queue.
+ String notifiedTopic = topicDeliveringTo("backup-notify-topic", notifiedQueue,
+ "{\"EventType\":[\"BACKUP_JOB\"],\"State\":[\"COMPLETED\"]}");
+
+ // The quiet vault's job is started first. Completions run on one scheduler thread in
+ // start order, so by the time the notified vault's message has arrived, any publish
+ // the quiet vault's completion was going to make has already been made.
+ String quietJob = startJobInNotifiedVault("notify-quiet-vault", quietTopic, "BACKUP_JOB_FAILED");
+ String jid = startJobInNotifiedVault("notify-vault", notifiedTopic, "BACKUP_JOB_COMPLETED");
+
+ String[] body = new String[1];
+ await().atMost(Duration.ofSeconds(10)).pollInterval(Duration.ofMillis(100))
+ .until(() -> (body[0] = receiveBody(notifiedQueue)) != null && !body[0].isEmpty());
+
+ awaitJobCompleted(jid);
+ String rpArn = given().header("Authorization", AUTH)
+ .when().get("/backup-jobs/" + jid).then().statusCode(200)
+ .extract().path("RecoveryPointArn");
+
+ JsonPath envelope = new JsonPath(body[0]);
+ assertEquals(notifiedTopic, envelope.getString("TopicArn"));
+ assertEquals("Notification from AWS Backup", envelope.getString("Subject"));
+ // AWS's documented body, which ends at the job ID and does not name the vault.
+ assertEquals("An AWS Backup job was completed successfully. Recovery point ARN: " + rpArn
+ + ". Resource ARN : " + RESOURCE_ARN + ". BackupJob ID : " + jid, envelope.getString("Message"));
+ long created = given().header("Authorization", AUTH)
+ .when().get("/backup-jobs/" + jid).then().statusCode(200)
+ .extract().jsonPath().getLong("CreationDate");
+ assertEquals(Map.of(
+ "EventType", Map.of("Type", "String", "Value", "BACKUP_JOB"),
+ "State", Map.of("Type", "String", "Value", "COMPLETED"),
+ "AccountId", Map.of("Type", "String", "Value", "000000000000"),
+ "Id", Map.of("Type", "String", "Value", jid),
+ "StartTime", Map.of("Type", "String", "Value", Instant.ofEpochSecond(created).toString())),
+ envelope.getMap("MessageAttributes"));
+
+ awaitJobCompleted(quietJob);
+ assertThat(receiveBody(quietQueue), anyOf(nullValue(), emptyString()));
+ }
+
+ @Test
+ @Order(171)
+ void aFailedPublishIsLoggedWithItsStackTraceAndDoesNotUndoTheJob() {
+ List<LogRecord> records = new CopyOnWriteArrayList<>();
+ Handler capture = new Handler() {
+ @Override public void publish(LogRecord r) { records.add(r); }
+ @Override public void flush() { }
+ @Override public void close() { }
+ };
+ Logger log = Logger.getLogger(BackupService.class.getName());
+ log.addHandler(capture);
+ try {
+ String missingTopic = "arn:aws:sns:us-east-1:000000000000:backup-notify-no-such-topic";
+ String jid = startJobInNotifiedVault("notify-broken-vault", missingTopic, "BACKUP_JOB_COMPLETED");
+ awaitJobCompleted(jid);
+ await().atMost(Duration.ofSeconds(10)).until(() -> records.stream()
+ .anyMatch(r -> r.getLevel().intValue() >= Level.WARNING.intValue()
+ && String.valueOf(r.getMessage()).contains("BACKUP_JOB_COMPLETED")));
+ assertThat("the warning carries the exception, not just its message",
+ records.stream().filter(r -> String.valueOf(r.getMessage()).contains("BACKUP_JOB_COMPLETED"))
+ .allMatch(r -> r.getThrown() != null));
+ } finally {
+ log.removeHandler(capture);
+ }
+ }
+
+ @Test
+ @Order(172)
+ void aJobWhoseVaultWasReplacedDoesNotNotifyTheReplacement() {
+ String replacementQueue = createQueue("backup-notify-replacement-queue");
+ String replacementTopic = topicDeliveringTo("backup-notify-replacement-topic", replacementQueue);
+ String sentinelQueue = createQueue("backup-notify-sentinel-queue");
+ String sentinelTopic = topicDeliveringTo("backup-notify-sentinel-topic", sentinelQueue);
+ String vault = "notify-replaced-vault";
+
+ // No waiting for the clock: the original and its replacement are normally created in
+ // the same second, and must still be told apart.
+ given().header("Authorization", AUTH).contentType("application/json").body("{}")
+ .when().put("/backup-vaults/" + vault).then().statusCode(200);
+ // The incarnation id is Floci-internal and must not leak into the AWS response.
+ given().header("Authorization", AUTH)
+ .when().get("/backup-vaults/" + vault).then().statusCode(200)
+ .body("$", not(hasKey("FlociVaultId")));
+ given().header("Authorization", AUTH)
+ .when().get("/backup-vaults/").then().statusCode(200)
+ .body(not(containsString("FlociVaultId")));
+
+ String oldJob = given().header("Authorization", AUTH).contentType("application/json")
+ .body("""
+ {"BackupVaultName": "%s", "ResourceArn": "%s", "IamRoleArn": "%s"}
+ """.formatted(vault, RESOURCE_ARN, IAM_ROLE))
+ .when().put("/backup-jobs").then().statusCode(200).extract().path("BackupJobId");
+ given().header("Authorization", AUTH)
+ .when().delete("/backup-vaults/" + vault).then().statusCode(204);
+ // Recreates the name and points it at a topic the old job must never reach.
+ startJobInNotifiedVault(vault, replacementTopic, "BACKUP_JOB_COMPLETED");
+ // Completions run in start order on one thread: once this one is delivered, the
+ // old job's completion, and any publish it was going to make, has already happened.
+ startJobInNotifiedVault("notify-sentinel-vault", sentinelTopic, "BACKUP_JOB_COMPLETED");
+
+ await().atMost(Duration.ofSeconds(10)).pollInterval(Duration.ofMillis(100))
+ .until(() -> { String b = receiveBody(sentinelQueue); return b != null && !b.isEmpty(); });
+ awaitJobCompleted(oldJob);
+
+ // The replacement's own job does publish; the old job must not have published too.
+ String first = receiveBody(replacementQueue);
+ assertThat(first, not(containsString("BackupJob ID : " + oldJob)));
+ String second = receiveBody(replacementQueue);
+ assertThat(second == null ? "" : second, not(containsString("BackupJob ID : " + oldJob)));
+ }
+
}