Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): re-check TTL under item lock before sweeper deletes (#3562)
Niels Kootstraf1bbe12d62f9
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex f13446e755..f13d140dc2 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -1909,8 +1909,15 @@ public class DynamoDbService implements ResourceProvider {}}+ record ExpiredTableScan(String rawKey, String accountId, String storageKey, String region,+ TableDefinition table, List<String> itemKeys) {}+void deleteExpiredItems() {- int totalDeleted = 0;+ deleteScannedItems(scanExpiredItems());+ }++ List<ExpiredTableScan> scanExpiredItems() {+ List<ExpiredTableScan> scans = new ArrayList<>();// Runs with no request scope and must sweep every account's tables, not just the// default one — scanAllAccountsRaw()'s key matches itemsByTable's directly.Map<String, TableDefinition> allTables;@@ -1927,7 +1934,9 @@ public class DynamoDbService implements ResourceProvider {continue;}var items = itemsByTable.get(rawKey);- if (items == null) continue;+ if (items == null) {+ continue;+ }List<String> expiredKeys = items.entrySet().stream().filter(e -> isExpired(e.getValue(), table))@@ -1940,21 +1949,46 @@ public class DynamoDbService implements ResourceProvider {String accountId = slash >= 0 ? rawKey.substring(0, slash) : null;String storageKey = slash >= 0 ? rawKey.substring(slash + 1) : rawKey;String region = storageKey.split("::", 2)[0];- for (String itemKey : expiredKeys) {- JsonNode removed = items.remove(itemKey);- if (removed != null) {- if (streamService != null) {- streamService.captureEvent(table.getTableName(), "REMOVE", removed, null, table, region);- }- if (kinesisForwarder != null) {- // Out of request scope here: pass the table owner's account explicitly so the CDC- // record lands in the owner's stream, not the default account's same-named stream.- kinesisForwarder.forward("REMOVE", removed, null, table, region, accountId);+ scans.add(new ExpiredTableScan(rawKey, accountId, storageKey, region, table, expiredKeys));+ }+ return scans;+ }++ void deleteScannedItems(List<ExpiredTableScan> scans) {+ int totalDeleted = 0;+ for (ExpiredTableScan scan : scans) {+ ConcurrentSkipListMap<String, JsonNode> items = itemsByTable.get(scan.rawKey());+ if (items == null) {+ continue;+ }++ int deletedForTable = 0;+ for (String itemKey : scan.itemKeys()) {+ JsonNode removed = withScopedItemLock(scan.rawKey(), itemKey, () -> {+ JsonNode current = items.get(itemKey);+ if (current == null || !isExpired(current, scan.table())) {+ return null;}+ return items.remove(itemKey);+ });+ if (removed == null) {+ continue;+ }+ deletedForTable++;+ if (streamService != null) {+ streamService.captureEvent(scan.table().getTableName(), "REMOVE", removed, null,+ scan.table(), scan.region());+ }+ if (kinesisForwarder != null) {+ // Out of request scope here: pass the table owner's account explicitly so the CDC+ // record lands in the owner's stream, not the default account's same-named stream.+ kinesisForwarder.forward("REMOVE", removed, null, scan.table(), scan.region(), scan.accountId());}}- persistItemsForAccount(accountId, storageKey, items);- totalDeleted += expiredKeys.size();+ if (deletedForTable > 0) {+ persistItemsForAccount(scan.accountId(), scan.storageKey(), items);+ totalDeleted += deletedForTable;+ }}if (totalDeleted > 0) {LOG.infov("TTL sweeper removed {0} expired items", totalDeleted);@@ -3071,8 +3105,13 @@ public class DynamoDbService implements ResourceProvider {}private ReentrantLock lockFor(String storageKey, String itemKey) {+ return lockForScopedKey(scopedItemsKey(storageKey), itemKey);+ }++ // The sweeper has no request scope, so it locks with the raw account-scoped key instead of scopedItemsKey.+ private ReentrantLock lockForScopedKey(String scopedKey, String itemKey) {return itemLocks- .computeIfAbsent(scopedItemsKey(storageKey), k -> new ConcurrentHashMap<>())+ .computeIfAbsent(scopedKey, k -> new ConcurrentHashMap<>()).computeIfAbsent(itemKey, k -> new ReentrantLock());}@@ -3096,6 +3135,16 @@ public class DynamoDbService implements ResourceProvider {}}+ private <T> T withScopedItemLock(String scopedKey, String itemKey, Supplier<T> body) {+ ReentrantLock lock = lockForScopedKey(scopedKey, itemKey);+ lock.lock();+ try {+ return body.get();+ } finally {+ lock.unlock();+ }+ }+// AWS words a key rejection by the surface the key arrived on. A PutItem item body// names the mismatched types, a Key argument and a BatchWriteItem entry report a// schema mismatch instead. An empty key value is worded the same on every surface.diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.javaindex cc0cf2afa3..64f9bedae7 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java@@ -9,8 +9,10 @@ import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;+import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;import io.github.hectorvent.floci.services.kinesis.KinesisService;import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;@@ -198,4 +200,70 @@ class DynamoDbTtlForwardingTest {assertNull(reloaded.getItem("TtlTable", key, REGION), "expired removal must be persisted");}}++ // A write landing between scanExpiredItems() and deleteScannedItems() must not be lost.+ @Test+ void ttlSweep_preserves_item_updated_between_scan_and_delete() throws Exception {+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();+ DynamoDbStreamService streamService = new DynamoDbStreamService(mapper, tableStore);+ KinesisService kinesis = realKinesis();+ KinesisStreamingForwarder forwarder = new KinesisStreamingForwarder(kinesis, mapper);+ DynamoDbService svc = new DynamoDbService(+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"),+ streamService, forwarder);++ seedTtlTable(svc, 2);+ TableDefinition table = tableStore.get("us-east-1::TtlTable").orElseThrow();+ StreamDescription sd = streamService.enableStream(+ table.getTableName(), table.getTableArn(), "NEW_AND_OLD_IMAGES", REGION);+ KinesisStream stream = kinesis.createStream("ttl-race-stream", 1, REGION);+ table.getKinesisStreamingDestinations().add(new KinesisStreamingDestination(stream.getStreamArn()));++ List<DynamoDbService.ExpiredTableScan> scans = svc.scanExpiredItems();+ int candidateCount = scans.stream().mapToInt(scan -> scan.itemKeys().size()).sum();+ assertEquals(2, candidateCount, "both expired rows must be scan candidates");++ // Lands after the scan but before the delete step runs against that stale snapshot.+ ObjectNode refreshed = mapper.createObjectNode();+ refreshed.set("pk", stringAttr("row-0"));+ refreshed.set("expireAt", numberAttr(Instant.now().getEpochSecond() + 3600));+ refreshed.set("marker", stringAttr("updated-during-sweep"));+ svc.putItem("TtlTable", refreshed, REGION);++ svc.deleteScannedItems(scans);+ assertTrue(forwarder.awaitIdle(Duration.ofSeconds(5)), "CDC forwards should drain promptly");++ ObjectNode keyRow0 = mapper.createObjectNode();+ keyRow0.set("pk", stringAttr("row-0"));+ JsonNode row0 = svc.getItem("TtlTable", keyRow0, REGION);+ assertNotNull(row0, "the concurrently updated item must survive the sweep");+ assertEquals("updated-during-sweep", row0.get("marker").get("S").asText());++ ObjectNode keyRow1 = mapper.createObjectNode();+ keyRow1.set("pk", stringAttr("row-1"));+ assertNull(svc.getItem("TtlTable", keyRow1, REGION), "the genuinely expired item must still be removed");++ // The survivor's own update also lands on both destinations as a MODIFY; only the+ // REMOVE side is what this test is about, so isolate that event before asserting on it.+ String iterator = streamService.getShardIterator(+ sd.getStreamArn(), DynamoDbStreamService.SHARD_ID, "TRIM_HORIZON", null);+ DynamoDbStreamService.GetRecordsResult pulled = streamService.getRecords(iterator, 1000);+ List<DynamoDbStreamRecord> removes = pulled.records().stream()+ .filter(record -> "REMOVE".equals(record.getEventName()))+ .toList();+ assertEquals(1, removes.size(), "a skipped delete must not emit a stream REMOVE record");+ assertEquals("row-1", removes.get(0).getKeys().get("pk").get("S").asText());++ List<KinesisRecord> drained = drain(kinesis, "ttl-race-stream");+ List<JsonNode> kinesisRemoves = new ArrayList<>();+ for (KinesisRecord record : drained) {+ JsonNode payload = mapper.readTree(record.getData());+ if ("REMOVE".equals(payload.get("eventName").asText())) {+ kinesisRemoves.add(payload);+ }+ }+ assertEquals(1, kinesisRemoves.size(), "a skipped delete must not forward a Kinesis REMOVE record");+ assertEquals("row-1", kinesisRemoves.get(0).get("dynamodb").get("Keys").get("pk").get("S").asText());+ }}