Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(bedrock-agentcore): expire remembered delete client tokens (#4477)

Niels Kootstrafe55cf0d0680
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlService.java b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlService.java
index bb12f4572f..a7fbcc8e10 100644
--- a/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlService.java
@@ -15,14 +15,13 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.time.Clock;
import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
import java.util.regex.Pattern;
@@ -49,15 +48,16 @@ public class BedrockAgentCoreControlService {
private final RegionResolver regionResolver;
private final BedrockAgentCoreIdentityService identityService;
// clientToken idempotency: tokens of completed deletes, so a replayed delete succeeds
- // instead of 404ing. In-memory only (reset on restart) — sufficient for an emulator.
- private final Set<String> deletedRuntimeTokens = ConcurrentHashMap.newKeySet();
- private final Set<String> deletedEndpointTokens = ConcurrentHashMap.newKeySet();
+ // instead of 404ing. In-memory only (reset on restart), and each entry expires after
+ // DeletedTokenLedger.TTL, after which a replay 404s like an unknown token.
+ private final DeletedTokenLedger deletedRuntimeTokens;
+ private final DeletedTokenLedger deletedEndpointTokens;
@Inject
public BedrockAgentCoreControlService(StorageFactory storageFactory, RegionResolver regionResolver,
- BedrockAgentCoreIdentityService identityService) {
+ BedrockAgentCoreIdentityService identityService, Clock clock) {
this(storageFactory.create("bedrockagentcore", "bedrock-agentcore-runtimes.json",
- new TypeReference<Map<String, AgentRuntime>>() {}), regionResolver, identityService);
+ new TypeReference<Map<String, AgentRuntime>>() {}), regionResolver, identityService, clock);
}
BedrockAgentCoreControlService(StorageBackend<String, AgentRuntime> storage, RegionResolver regionResolver) {
@@ -66,9 +66,16 @@ public class BedrockAgentCoreControlService {
BedrockAgentCoreControlService(StorageBackend<String, AgentRuntime> storage, RegionResolver regionResolver,
BedrockAgentCoreIdentityService identityService) {
+ this(storage, regionResolver, identityService, Clock.systemUTC());
+ }
+
+ BedrockAgentCoreControlService(StorageBackend<String, AgentRuntime> storage, RegionResolver regionResolver,
+ BedrockAgentCoreIdentityService identityService, Clock clock) {
this.storage = storage;
this.regionResolver = regionResolver;
this.identityService = identityService;
+ this.deletedRuntimeTokens = new DeletedTokenLedger(clock);
+ this.deletedEndpointTokens = new DeletedTokenLedger(clock);
}
public AgentRuntime createAgentRuntime(String name, JsonNode artifact, JsonNode networkConfiguration,
@@ -204,7 +211,7 @@ public class BedrockAgentCoreControlService {
AgentRuntime runtime = found.get();
storage.delete(key(region, id));
if (clientToken != null) {
- deletedRuntimeTokens.add(tokenKey(region, clientToken));
+ deletedRuntimeTokens.record(tokenKey(region, clientToken));
}
runtime.setStatus(STATUS_DELETING);
LOG.infov("Deleted AgentCore runtime {0}", id);
@@ -381,7 +388,7 @@ public class BedrockAgentCoreControlService {
AgentRuntimeEndpoint endpoint = found.get();
runtime.getEndpoints().removeIf(e -> name.equals(e.getName()));
if (clientToken != null) {
- deletedEndpointTokens.add(tokenKey(region, clientToken));
+ deletedEndpointTokens.record(tokenKey(region, clientToken));
}
endpoint.setStatus(STATUS_DELETING);
storage.put(key(region, runtimeId), runtime);
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryService.java b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryService.java
index be4f81c02a..a446d755e5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryService.java
@@ -11,13 +11,12 @@ import io.github.hectorvent.floci.services.bedrockagentcorecontrol.model.Memory;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
+import java.time.Clock;
import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
/** CRUD for AgentCore memory resources. Metadata registry only. */
@@ -32,17 +31,25 @@ public class BedrockAgentCoreMemoryService {
private final StorageBackend<String, Memory> storage;
private final RegionResolver regionResolver;
- private final Set<String> deletedTokens = ConcurrentHashMap.newKeySet();
+ // clientToken idempotency: tokens of completed deletes, so a replayed delete succeeds
+ // instead of 404ing. In-memory only (reset on restart), and each entry expires after
+ // DeletedTokenLedger.TTL, after which a replay 404s like an unknown token.
+ private final DeletedTokenLedger deletedTokens;
@Inject
- public BedrockAgentCoreMemoryService(StorageFactory storageFactory, RegionResolver regionResolver) {
+ public BedrockAgentCoreMemoryService(StorageFactory storageFactory, RegionResolver regionResolver, Clock clock) {
this(storageFactory.create("bedrockagentcore", "bedrock-agentcore-memories.json",
- new TypeReference<Map<String, Memory>>() {}), regionResolver);
+ new TypeReference<Map<String, Memory>>() {}), regionResolver, clock);
}
BedrockAgentCoreMemoryService(StorageBackend<String, Memory> storage, RegionResolver regionResolver) {
+ this(storage, regionResolver, Clock.systemUTC());
+ }
+
+ BedrockAgentCoreMemoryService(StorageBackend<String, Memory> storage, RegionResolver regionResolver, Clock clock) {
this.storage = storage;
this.regionResolver = regionResolver;
+ this.deletedTokens = new DeletedTokenLedger(clock);
}
public Memory create(String name, Integer eventExpiryDuration, String description,
@@ -132,7 +139,7 @@ public class BedrockAgentCoreMemoryService {
Memory memory = found.get();
storage.delete(key(region, id));
if (clientToken != null) {
- deletedTokens.add(region + " " + clientToken);
+ deletedTokens.record(region + " " + clientToken);
}
memory.setStatus(STATUS_DELETING);
return memory;
diff --git a/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/DeletedTokenLedger.java b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/DeletedTokenLedger.java
new file mode 100644
index 0000000000..0b6b2fb6d8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/DeletedTokenLedger.java
@@ -0,0 +1,70 @@
+package io.github.hectorvent.floci.services.bedrockagentcorecontrol;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Remembers the clientTokens of completed deletes so a replayed delete on an
+ * already-deleted resource returns the same DELETING marker instead of 404ing,
+ * the way AWS's control planes do.
+ *
+ * <p>AWS does not document an idempotency window for the AgentCore delete
+ * operations themselves, so this follows the window AWS documents for a
+ * clientToken elsewhere (for example DynamoDB's ImportTable and SageMaker
+ * APIs): valid for 8 hours after the first request that used it, after which
+ * a repeat is treated as a new request. A token past that window behaves as if
+ * it were never recorded, both when it is looked up and, so this does not
+ * grow without bound, whenever another token is recorded. Tokens are kept
+ * in the order they were first recorded, one entry per token, so purge only
+ * visits the expired ones at the front instead of scanning every live token.
+ */
+final class DeletedTokenLedger {
+
+ static final Duration TTL = Duration.ofHours(8);
+
+ private final LinkedHashMap<String, Instant> recordedAt = new LinkedHashMap<>();
+ private final Clock clock;
+
+ DeletedTokenLedger(Clock clock) {
+ this.clock = clock;
+ }
+
+ /** Records {@code key} as deleted unless it is already live, first purging any tokens past their TTL. */
+ synchronized void record(String key) {
+ purgeExpired();
+ // Keep the first timestamp of a live token: the window runs from the first request that
+ // used it. contains drops an expired entry the front purge could not reach, for example
+ // after the wall clock stepped back, so reusing it starts a fresh window.
+ if (!contains(key)) {
+ recordedAt.put(key, Instant.now(clock));
+ }
+ }
+
+ /** Whether {@code key} was recorded within its TTL. An expired entry is dropped and treated as absent. */
+ synchronized boolean contains(String key) {
+ Instant at = recordedAt.get(key);
+ if (at == null) {
+ return false;
+ }
+ if (isExpired(at)) {
+ recordedAt.remove(key);
+ return false;
+ }
+ return true;
+ }
+
+ private void purgeExpired() {
+ Iterator<Map.Entry<String, Instant>> oldestFirst = recordedAt.entrySet().iterator();
+ while (oldestFirst.hasNext() && isExpired(oldestFirst.next().getValue())) {
+ oldestFirst.remove();
+ }
+ }
+
+ private boolean isExpired(Instant at) {
+ return at.plus(TTL).isBefore(Instant.now(clock));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlServiceTest.java
index 91e9b8d3f3..7dcf13243b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreControlServiceTest.java
@@ -9,9 +9,11 @@ import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.services.bedrockagentcorecontrol.model.AgentRuntime;
import io.github.hectorvent.floci.services.bedrockagentcorecontrol.model.AgentRuntimeEndpoint;
import io.github.hectorvent.floci.services.bedrockagentcorecontrol.model.AgentRuntimeVersion;
+import io.github.hectorvent.floci.testing.MutableClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.time.Duration;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutorService;
@@ -28,12 +30,13 @@ class BedrockAgentCoreControlServiceTest {
private static final String REGION = "us-east-1";
private final ObjectMapper mapper = new ObjectMapper();
+ private final MutableClock clock = new MutableClock();
private BedrockAgentCoreControlService service;
@BeforeEach
void setUp() {
service = new BedrockAgentCoreControlService(
- new InMemoryStorage<>(), new RegionResolver(REGION, "000000000000"));
+ new InMemoryStorage<>(), new RegionResolver(REGION, "000000000000"), null, clock);
}
private ObjectNode artifact() {
@@ -217,6 +220,76 @@ class BedrockAgentCoreControlServiceTest {
() -> service.deleteAgentRuntime(id, "other", REGION)).getHttpStatus());
}
+ @Test
+ void deleteRuntimeTokenExpiresAfterWindowReturns404() {
+ AgentRuntime rt = create("myAgent");
+ String id = rt.getAgentRuntimeId();
+ assertEquals("DELETING", service.deleteAgentRuntime(id, "del-1", REGION).getStatus());
+
+ clock.advance(DeletedTokenLedger.TTL.plusSeconds(1));
+
+ assertEquals(404, assertThrows(AwsException.class,
+ () -> service.deleteAgentRuntime(id, "del-1", REGION)).getHttpStatus());
+ }
+
+ @Test
+ void deleteRuntimeTokenCanBeReusedAfterItExpires() {
+ // Once a token's record expires it must behave as if it had never been recorded,
+ // not merely as "expired but still occupying the map": a later delete that reuses
+ // the same token value records it again, and that fresh record is honored.
+ AgentRuntime first = create("firstAgent");
+ String firstId = first.getAgentRuntimeId();
+ assertEquals("DELETING", service.deleteAgentRuntime(firstId, "reused-token", REGION).getStatus());
+
+ clock.advance(DeletedTokenLedger.TTL.plusSeconds(1));
+
+ AgentRuntime second = create("secondAgent");
+ String secondId = second.getAgentRuntimeId();
+ assertEquals("DELETING", service.deleteAgentRuntime(secondId, "reused-token", REGION).getStatus());
+
+ // The replay right after is still within the fresh token's own window.
+ assertEquals("DELETING", service.deleteAgentRuntime(secondId, "reused-token", REGION).getStatus());
+ // The old runtime's id is unrelated to the reused token and stays gone.
+ assertEquals(404, assertThrows(AwsException.class,
+ () -> service.deleteAgentRuntime(firstId, "some-other-token", REGION)).getHttpStatus());
+ }
+
+ @Test
+ void deleteRuntimeTokenReusedWithinItsWindowStillExpiresFromItsFirstUse() {
+ // The window runs from the first request that used the token, so reusing it does
+ // not extend the window.
+ String firstId = create("firstAgent").getAgentRuntimeId();
+ assertEquals("DELETING", service.deleteAgentRuntime(firstId, "reused-token", REGION).getStatus());
+
+ clock.advance(Duration.ofHours(7));
+ String secondId = create("secondAgent").getAgentRuntimeId();
+ assertEquals("DELETING", service.deleteAgentRuntime(secondId, "reused-token", REGION).getStatus());
+
+ clock.advance(Duration.ofHours(1).plusSeconds(1));
+
+ assertEquals(404, assertThrows(AwsException.class,
+ () -> service.deleteAgentRuntime(secondId, "reused-token", REGION)).getHttpStatus());
+ }
+
+ @Test
+ void deleteRuntimeTokenReusedAfterExpiryIsHonouredEvenWhenTheClockMovedBack() {
+ // A wall clock stepping back can leave an expired token behind a live one, where the
+ // front purge cannot reach it. Reusing that token must still start a fresh window.
+ clock.advance(Duration.ofHours(2));
+ String liveId = create("liveAgent").getAgentRuntimeId();
+ assertEquals("DELETING", service.deleteAgentRuntime(liveId, "live-token", REGION).getStatus());
+
+ clock.advance(Duration.ofHours(-3));
+ String staleId = create("staleAgent").getAgentRuntimeId();
+ assertEquals("DELETING", service.deleteAgentRuntime(staleId, "stale-token", REGION).getStatus());
+
+ clock.advance(DeletedTokenLedger.TTL.plusSeconds(1));
+ String reusedId = create("reusedAgent").getAgentRuntimeId();
+ assertEquals("DELETING", service.deleteAgentRuntime(reusedId, "stale-token", REGION).getStatus());
+
+ assertEquals("DELETING", service.deleteAgentRuntime(reusedId, "stale-token", REGION).getStatus());
+ }
+
@Test
void deleteRemovesAndReportsDeleting() {
AgentRuntime rt = create("myAgent");
@@ -286,4 +359,31 @@ class BedrockAgentCoreControlServiceTest {
assertEquals(409, assertThrows(AwsException.class,
() -> service.createEndpoint(id, "DEFAULT", null, null, null, REGION)).getHttpStatus());
}
+
+ @Test
+ void deleteEndpointIsIdempotentByClientToken() {
+ AgentRuntime rt = create("myAgent");
+ String id = rt.getAgentRuntimeId();
+ service.createEndpoint(id, "prod", null, null, null, REGION);
+
+ assertEquals("DELETING", service.deleteEndpoint(id, "prod", "ep-del-1", REGION).getStatus());
+ // Replayed delete with the same token succeeds instead of 404ing.
+ assertEquals("DELETING", service.deleteEndpoint(id, "prod", "ep-del-1", REGION).getStatus());
+ // A different token against the now-missing endpoint still 404s.
+ assertEquals(404, assertThrows(AwsException.class,
+ () -> service.deleteEndpoint(id, "prod", "other", REGION)).getHttpStatus());
+ }
+
+ @Test
+ void deleteEndpointTokenExpiresAfterWindowReturns404() {
+ AgentRuntime rt = create("myAgent");
+ String id = rt.getAgentRuntimeId();
+ service.createEndpoint(id, "prod", null, null, null, REGION);
+ assertEquals("DELETING", service.deleteEndpoint(id, "prod", "ep-del-1", REGION).getStatus());
+
+ clock.advance(DeletedTokenLedger.TTL.plusSeconds(1));
+
+ assertEquals(404, assertThrows(AwsException.class,
+ () -> service.deleteEndpoint(id, "prod", "ep-del-1", REGION)).getHttpStatus());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryServiceTest.java
index 0b3413d50c..3575dab632 100644
--- a/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/bedrockagentcorecontrol/BedrockAgentCoreMemoryServiceTest.java
@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.services.bedrockagentcorecontrol.model.Memory;
+import io.github.hectorvent.floci.testing.MutableClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -21,12 +22,13 @@ class BedrockAgentCoreMemoryServiceTest {
private static final String KEY_ARN = "arn:aws:kms:us-east-1:000000000000:key/mem-key";
private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/mem-role";
+ private final MutableClock clock = new MutableClock();
private BedrockAgentCoreMemoryService service;
@BeforeEach
void setUp() {
service = new BedrockAgentCoreMemoryService(
- new InMemoryStorage<>(), new RegionResolver(REGION, "000000000000"));
+ new InMemoryStorage<>(), new RegionResolver(REGION, "000000000000"), clock);
}
private Memory create(String name) {
@@ -102,6 +104,30 @@ class BedrockAgentCoreMemoryServiceTest {
assertEquals(ROLE_ARN, after.getMemoryExecutionRoleArn());
}
+ @Test
+ void deleteIsIdempotentByClientToken() {
+ Memory memory = create("myMemory");
+ String id = memory.getMemoryId();
+ assertEquals("DELETING", service.delete(id, "del-1", REGION).getStatus());
+ // Replayed delete with the same token succeeds instead of 404ing.
+ assertEquals("DELETING", service.delete(id, "del-1", REGION).getStatus());
+ // A different token against the now-missing id still 404s.
+ assertEquals(404, assertThrows(AwsException.class,
+ () -> service.delete(id, "other", REGION)).getHttpStatus());
+ }
+
+ @Test
+ void deleteTokenExpiresAfterWindowReturns404() {
+ Memory memory = create("myMemory");
+ String id = memory.getMemoryId();
+ assertEquals("DELETING", service.delete(id, "del-1", REGION).getStatus());
+
+ clock.advance(DeletedTokenLedger.TTL.plusSeconds(1));
+
+ assertEquals(404, assertThrows(AwsException.class,
+ () -> service.delete(id, "del-1", REGION)).getHttpStatus());
+ }
+
@Test
void tagOperationsRoundTripThroughTheArn() {
Memory memory = create("tagMemory");