Light, fluffy, and always free - The AWS Local Emulator alternative
fix(bedrock-agentcore): expire remembered delete client tokens (#4477)
Niels Kootstrafe55cf0d0680
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlService.java b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlService.javaindex bb12f4572f..a7fbcc8e10 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlService.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlService.java@@ -15,14 +15,13 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.time.Clock;import java.time.Instant;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Optional;-import java.util.Set;import java.util.UUID;-import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ThreadLocalRandom;import java.util.regex.Pattern;@@ -49,15 +48,16 @@ public class BedrockAgentCoreControlService {private final RegionResolver regionResolver;private final BedrockAgentCoreIdentityService identityService;// clientToken idempotency: tokens of completed deletes, so a replayed delete succeeds- // instead of 404ing. In-memory only (reset on restart) — sufficient for an emulator.- private final Set<String> deletedRuntimeTokens = ConcurrentHashMap.newKeySet();- private final Set<String> deletedEndpointTokens = ConcurrentHashMap.newKeySet();+ // instead of 404ing. In-memory only (reset on restart), and each entry expires after+ // DeletedTokenLedger.TTL, after which a replay 404s like an unknown token.+ private final DeletedTokenLedger deletedRuntimeTokens;+ private final DeletedTokenLedger deletedEndpointTokens;@Injectpublic BedrockAgentCoreControlService(StorageFactory storageFactory, RegionResolver regionResolver,- BedrockAgentCoreIdentityService identityService) {+ BedrockAgentCoreIdentityService identityService, Clock clock) {this(storageFactory.create("bedrockagentcore", "bedrock-agentcore-runtimes.json",- new TypeReference<Map<String, AgentRuntime>>() {}), regionResolver, identityService);+ new TypeReference<Map<String, AgentRuntime>>() {}), regionResolver, identityService, clock);}BedrockAgentCoreControlService(StorageBackend<String, AgentRuntime> storage, RegionResolver regionResolver) {@@ -66,9 +66,16 @@ public class BedrockAgentCoreControlService {BedrockAgentCoreControlService(StorageBackend<String, AgentRuntime> storage, RegionResolver regionResolver,BedrockAgentCoreIdentityService identityService) {+ this(storage, regionResolver, identityService, Clock.systemUTC());+ }++ BedrockAgentCoreControlService(StorageBackend<String, AgentRuntime> storage, RegionResolver regionResolver,+ BedrockAgentCoreIdentityService identityService, Clock clock) {this.storage = storage;this.regionResolver = regionResolver;this.identityService = identityService;+ this.deletedRuntimeTokens = new DeletedTokenLedger(clock);+ this.deletedEndpointTokens = new DeletedTokenLedger(clock);}public AgentRuntime createAgentRuntime(String name, JsonNode artifact, JsonNode networkConfiguration,@@ -204,7 +211,7 @@ public class BedrockAgentCoreControlService {AgentRuntime runtime = found.get();storage.delete(key(region, id));if (clientToken != null) {- deletedRuntimeTokens.add(tokenKey(region, clientToken));+ deletedRuntimeTokens.record(tokenKey(region, clientToken));}runtime.setStatus(STATUS_DELETING);LOG.infov("Deleted AgentCore runtime {0}", id);@@ -381,7 +388,7 @@ public class BedrockAgentCoreControlService {AgentRuntimeEndpoint endpoint = found.get();runtime.getEndpoints().removeIf(e -> name.equals(e.getName()));if (clientToken != null) {- deletedEndpointTokens.add(tokenKey(region, clientToken));+ deletedEndpointTokens.record(tokenKey(region, clientToken));}endpoint.setStatus(STATUS_DELETING);storage.put(key(region, runtimeId), runtime);diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryService.java b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryService.javaindex be4f81c02a..a446d755e5 100644--- a/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryService.java+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryService.java@@ -11,13 +11,12 @@ import io.github.hectorvent.floci.services.bedrockagentcorecontrol.model.Memory;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import java.time.Clock;import java.time.Instant;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Optional;-import java.util.Set;-import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ThreadLocalRandom;/** CRUD for AgentCore memory resources. Metadata registry only. */@@ -32,17 +31,25 @@ public class BedrockAgentCoreMemoryService {private final StorageBackend<String, Memory> storage;private final RegionResolver regionResolver;- private final Set<String> deletedTokens = ConcurrentHashMap.newKeySet();+ // clientToken idempotency: tokens of completed deletes, so a replayed delete succeeds+ // instead of 404ing. In-memory only (reset on restart), and each entry expires after+ // DeletedTokenLedger.TTL, after which a replay 404s like an unknown token.+ private final DeletedTokenLedger deletedTokens;@Inject- public BedrockAgentCoreMemoryService(StorageFactory storageFactory, RegionResolver regionResolver) {+ public BedrockAgentCoreMemoryService(StorageFactory storageFactory, RegionResolver regionResolver, Clock clock) {this(storageFactory.create("bedrockagentcore", "bedrock-agentcore-memories.json",- new TypeReference<Map<String, Memory>>() {}), regionResolver);+ new TypeReference<Map<String, Memory>>() {}), regionResolver, clock);}BedrockAgentCoreMemoryService(StorageBackend<String, Memory> storage, RegionResolver regionResolver) {+ this(storage, regionResolver, Clock.systemUTC());+ }++ BedrockAgentCoreMemoryService(StorageBackend<String, Memory> storage, RegionResolver regionResolver, Clock clock) {this.storage = storage;this.regionResolver = regionResolver;+ this.deletedTokens = new DeletedTokenLedger(clock);}public Memory create(String name, Integer eventExpiryDuration, String description,@@ -132,7 +139,7 @@ public class BedrockAgentCoreMemoryService {Memory memory = found.get();storage.delete(key(region, id));if (clientToken != null) {- deletedTokens.add(region + " " + clientToken);+ deletedTokens.record(region + " " + clientToken);}memory.setStatus(STATUS_DELETING);return memory;diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/DeletedTokenLedger.java b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/DeletedTokenLedger.javanew file mode 100644index 0000000000..0b6b2fb6d8--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/DeletedTokenLedger.java@@ -0,0 +1,70 @@+package io.github.hectorvent.floci.services.bedrockagentcorecontrol;++import java.time.Clock;+import java.time.Duration;+import java.time.Instant;+import java.util.Iterator;+import java.util.LinkedHashMap;+import java.util.Map;++/**+ * Remembers the clientTokens of completed deletes so a replayed delete on an+ * already-deleted resource returns the same DELETING marker instead of 404ing,+ * the way AWS's control planes do.+ *+ * <p>AWS does not document an idempotency window for the AgentCore delete+ * operations themselves, so this follows the window AWS documents for a+ * clientToken elsewhere (for example DynamoDB's ImportTable and SageMaker+ * APIs): valid for 8 hours after the first request that used it, after which+ * a repeat is treated as a new request. A token past that window behaves as if+ * it were never recorded, both when it is looked up and, so this does not+ * grow without bound, whenever another token is recorded. Tokens are kept+ * in the order they were first recorded, one entry per token, so purge only+ * visits the expired ones at the front instead of scanning every live token.+ */+final class DeletedTokenLedger {++ static final Duration TTL = Duration.ofHours(8);++ private final LinkedHashMap<String, Instant> recordedAt = new LinkedHashMap<>();+ private final Clock clock;++ DeletedTokenLedger(Clock clock) {+ this.clock = clock;+ }++ /** Records {@code key} as deleted unless it is already live, first purging any tokens past their TTL. */+ synchronized void record(String key) {+ purgeExpired();+ // Keep the first timestamp of a live token: the window runs from the first request that+ // used it. contains drops an expired entry the front purge could not reach, for example+ // after the wall clock stepped back, so reusing it starts a fresh window.+ if (!contains(key)) {+ recordedAt.put(key, Instant.now(clock));+ }+ }++ /** Whether {@code key} was recorded within its TTL. An expired entry is dropped and treated as absent. */+ synchronized boolean contains(String key) {+ Instant at = recordedAt.get(key);+ if (at == null) {+ return false;+ }+ if (isExpired(at)) {+ recordedAt.remove(key);+ return false;+ }+ return true;+ }++ private void purgeExpired() {+ Iterator<Map.Entry<String, Instant>> oldestFirst = recordedAt.entrySet().iterator();+ while (oldestFirst.hasNext() && isExpired(oldestFirst.next().getValue())) {+ oldestFirst.remove();+ }+ }++ private boolean isExpired(Instant at) {+ return at.plus(TTL).isBefore(Instant.now(clock));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlServiceTest.javaindex 91e9b8d3f3..7dcf13243b 100644--- a/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlServiceTest.java@@ -9,9 +9,11 @@ import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.bedrockagentcorecontrol.model.AgentRuntime;import io.github.hectorvent.floci.services.bedrockagentcorecontrol.model.AgentRuntimeEndpoint;import io.github.hectorvent.floci.services.bedrockagentcorecontrol.model.AgentRuntimeVersion;+import io.github.hectorvent.floci.testing.MutableClock;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import java.time.Duration;import java.util.HashSet;import java.util.Set;import java.util.concurrent.ExecutorService;@@ -28,12 +30,13 @@ class BedrockAgentCoreControlServiceTest {private static final String REGION = "us-east-1";private final ObjectMapper mapper = new ObjectMapper();+ private final MutableClock clock = new MutableClock();private BedrockAgentCoreControlService service;@BeforeEachvoid setUp() {service = new BedrockAgentCoreControlService(- new InMemoryStorage<>(), new RegionResolver(REGION, "000000000000"));+ new InMemoryStorage<>(), new RegionResolver(REGION, "000000000000"), null, clock);}private ObjectNode artifact() {@@ -217,6 +220,76 @@ class BedrockAgentCoreControlServiceTest {() -> service.deleteAgentRuntime(id, "other", REGION)).getHttpStatus());}+ @Test+ void deleteRuntimeTokenExpiresAfterWindowReturns404() {+ AgentRuntime rt = create("myAgent");+ String id = rt.getAgentRuntimeId();+ assertEquals("DELETING", service.deleteAgentRuntime(id, "del-1", REGION).getStatus());++ clock.advance(DeletedTokenLedger.TTL.plusSeconds(1));++ assertEquals(404, assertThrows(AwsException.class,+ () -> service.deleteAgentRuntime(id, "del-1", REGION)).getHttpStatus());+ }++ @Test+ void deleteRuntimeTokenCanBeReusedAfterItExpires() {+ // Once a token's record expires it must behave as if it had never been recorded,+ // not merely as "expired but still occupying the map": a later delete that reuses+ // the same token value records it again, and that fresh record is honored.+ AgentRuntime first = create("firstAgent");+ String firstId = first.getAgentRuntimeId();+ assertEquals("DELETING", service.deleteAgentRuntime(firstId, "reused-token", REGION).getStatus());++ clock.advance(DeletedTokenLedger.TTL.plusSeconds(1));++ AgentRuntime second = create("secondAgent");+ String secondId = second.getAgentRuntimeId();+ assertEquals("DELETING", service.deleteAgentRuntime(secondId, "reused-token", REGION).getStatus());++ // The replay right after is still within the fresh token's own window.+ assertEquals("DELETING", service.deleteAgentRuntime(secondId, "reused-token", REGION).getStatus());+ // The old runtime's id is unrelated to the reused token and stays gone.+ assertEquals(404, assertThrows(AwsException.class,+ () -> service.deleteAgentRuntime(firstId, "some-other-token", REGION)).getHttpStatus());+ }++ @Test+ void deleteRuntimeTokenReusedWithinItsWindowStillExpiresFromItsFirstUse() {+ // The window runs from the first request that used the token, so reusing it does+ // not extend the window.+ String firstId = create("firstAgent").getAgentRuntimeId();+ assertEquals("DELETING", service.deleteAgentRuntime(firstId, "reused-token", REGION).getStatus());++ clock.advance(Duration.ofHours(7));+ String secondId = create("secondAgent").getAgentRuntimeId();+ assertEquals("DELETING", service.deleteAgentRuntime(secondId, "reused-token", REGION).getStatus());++ clock.advance(Duration.ofHours(1).plusSeconds(1));++ assertEquals(404, assertThrows(AwsException.class,+ () -> service.deleteAgentRuntime(secondId, "reused-token", REGION)).getHttpStatus());+ }++ @Test+ void deleteRuntimeTokenReusedAfterExpiryIsHonouredEvenWhenTheClockMovedBack() {+ // A wall clock stepping back can leave an expired token behind a live one, where the+ // front purge cannot reach it. Reusing that token must still start a fresh window.+ clock.advance(Duration.ofHours(2));+ String liveId = create("liveAgent").getAgentRuntimeId();+ assertEquals("DELETING", service.deleteAgentRuntime(liveId, "live-token", REGION).getStatus());++ clock.advance(Duration.ofHours(-3));+ String staleId = create("staleAgent").getAgentRuntimeId();+ assertEquals("DELETING", service.deleteAgentRuntime(staleId, "stale-token", REGION).getStatus());++ clock.advance(DeletedTokenLedger.TTL.plusSeconds(1));+ String reusedId = create("reusedAgent").getAgentRuntimeId();+ assertEquals("DELETING", service.deleteAgentRuntime(reusedId, "stale-token", REGION).getStatus());++ assertEquals("DELETING", service.deleteAgentRuntime(reusedId, "stale-token", REGION).getStatus());+ }+@Testvoid deleteRemovesAndReportsDeleting() {AgentRuntime rt = create("myAgent");@@ -286,4 +359,31 @@ class BedrockAgentCoreControlServiceTest {assertEquals(409, assertThrows(AwsException.class,() -> service.createEndpoint(id, "DEFAULT", null, null, null, REGION)).getHttpStatus());}++ @Test+ void deleteEndpointIsIdempotentByClientToken() {+ AgentRuntime rt = create("myAgent");+ String id = rt.getAgentRuntimeId();+ service.createEndpoint(id, "prod", null, null, null, REGION);++ assertEquals("DELETING", service.deleteEndpoint(id, "prod", "ep-del-1", REGION).getStatus());+ // Replayed delete with the same token succeeds instead of 404ing.+ assertEquals("DELETING", service.deleteEndpoint(id, "prod", "ep-del-1", REGION).getStatus());+ // A different token against the now-missing endpoint still 404s.+ assertEquals(404, assertThrows(AwsException.class,+ () -> service.deleteEndpoint(id, "prod", "other", REGION)).getHttpStatus());+ }++ @Test+ void deleteEndpointTokenExpiresAfterWindowReturns404() {+ AgentRuntime rt = create("myAgent");+ String id = rt.getAgentRuntimeId();+ service.createEndpoint(id, "prod", null, null, null, REGION);+ assertEquals("DELETING", service.deleteEndpoint(id, "prod", "ep-del-1", REGION).getStatus());++ clock.advance(DeletedTokenLedger.TTL.plusSeconds(1));++ assertEquals(404, assertThrows(AwsException.class,+ () -> service.deleteEndpoint(id, "prod", "ep-del-1", REGION)).getHttpStatus());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryServiceTest.javaindex 0b3413d50c..3575dab632 100644--- a/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryServiceTest.java@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.bedrockagentcorecontrol.model.Memory;+import io.github.hectorvent.floci.testing.MutableClock;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;@@ -21,12 +22,13 @@ class BedrockAgentCoreMemoryServiceTest {private static final String KEY_ARN = "arn:aws:kms:us-east-1:000000000000:key/mem-key";private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/mem-role";+ private final MutableClock clock = new MutableClock();private BedrockAgentCoreMemoryService service;@BeforeEachvoid setUp() {service = new BedrockAgentCoreMemoryService(- new InMemoryStorage<>(), new RegionResolver(REGION, "000000000000"));+ new InMemoryStorage<>(), new RegionResolver(REGION, "000000000000"), clock);}private Memory create(String name) {@@ -102,6 +104,30 @@ class BedrockAgentCoreMemoryServiceTest {assertEquals(ROLE_ARN, after.getMemoryExecutionRoleArn());}+ @Test+ void deleteIsIdempotentByClientToken() {+ Memory memory = create("myMemory");+ String id = memory.getMemoryId();+ assertEquals("DELETING", service.delete(id, "del-1", REGION).getStatus());+ // Replayed delete with the same token succeeds instead of 404ing.+ assertEquals("DELETING", service.delete(id, "del-1", REGION).getStatus());+ // A different token against the now-missing id still 404s.+ assertEquals(404, assertThrows(AwsException.class,+ () -> service.delete(id, "other", REGION)).getHttpStatus());+ }++ @Test+ void deleteTokenExpiresAfterWindowReturns404() {+ Memory memory = create("myMemory");+ String id = memory.getMemoryId();+ assertEquals("DELETING", service.delete(id, "del-1", REGION).getStatus());++ clock.advance(DeletedTokenLedger.TTL.plusSeconds(1));++ assertEquals(404, assertThrows(AwsException.class,+ () -> service.delete(id, "del-1", REGION)).getHttpStatus());+ }+@Testvoid tagOperationsRoundTripThroughTheArn() {Memory memory = create("tagMemory");