Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): re-check TTL under item lock before sweeper deletes (#3562)

Niels Kootstraf1bbe12d62f9
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index f13446e755..f13d140dc2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -1909,8 +1909,15 @@ public class DynamoDbService implements ResourceProvider {
}
}
+ record ExpiredTableScan(String rawKey, String accountId, String storageKey, String region,
+ TableDefinition table, List<String> itemKeys) {}
+
void deleteExpiredItems() {
- int totalDeleted = 0;
+ deleteScannedItems(scanExpiredItems());
+ }
+
+ List<ExpiredTableScan> scanExpiredItems() {
+ List<ExpiredTableScan> scans = new ArrayList<>();
// Runs with no request scope and must sweep every account's tables, not just the
// default one — scanAllAccountsRaw()'s key matches itemsByTable's directly.
Map<String, TableDefinition> allTables;
@@ -1927,7 +1934,9 @@ public class DynamoDbService implements ResourceProvider {
continue;
}
var items = itemsByTable.get(rawKey);
- if (items == null) continue;
+ if (items == null) {
+ continue;
+ }
List<String> expiredKeys = items.entrySet().stream()
.filter(e -> isExpired(e.getValue(), table))
@@ -1940,21 +1949,46 @@ public class DynamoDbService implements ResourceProvider {
String accountId = slash >= 0 ? rawKey.substring(0, slash) : null;
String storageKey = slash >= 0 ? rawKey.substring(slash + 1) : rawKey;
String region = storageKey.split("::", 2)[0];
- for (String itemKey : expiredKeys) {
- JsonNode removed = items.remove(itemKey);
- if (removed != null) {
- if (streamService != null) {
- streamService.captureEvent(table.getTableName(), "REMOVE", removed, null, table, region);
- }
- if (kinesisForwarder != null) {
- // Out of request scope here: pass the table owner's account explicitly so the CDC
- // record lands in the owner's stream, not the default account's same-named stream.
- kinesisForwarder.forward("REMOVE", removed, null, table, region, accountId);
+ scans.add(new ExpiredTableScan(rawKey, accountId, storageKey, region, table, expiredKeys));
+ }
+ return scans;
+ }
+
+ void deleteScannedItems(List<ExpiredTableScan> scans) {
+ int totalDeleted = 0;
+ for (ExpiredTableScan scan : scans) {
+ ConcurrentSkipListMap<String, JsonNode> items = itemsByTable.get(scan.rawKey());
+ if (items == null) {
+ continue;
+ }
+
+ int deletedForTable = 0;
+ for (String itemKey : scan.itemKeys()) {
+ JsonNode removed = withScopedItemLock(scan.rawKey(), itemKey, () -> {
+ JsonNode current = items.get(itemKey);
+ if (current == null || !isExpired(current, scan.table())) {
+ return null;
}
+ return items.remove(itemKey);
+ });
+ if (removed == null) {
+ continue;
+ }
+ deletedForTable++;
+ if (streamService != null) {
+ streamService.captureEvent(scan.table().getTableName(), "REMOVE", removed, null,
+ scan.table(), scan.region());
+ }
+ if (kinesisForwarder != null) {
+ // Out of request scope here: pass the table owner's account explicitly so the CDC
+ // record lands in the owner's stream, not the default account's same-named stream.
+ kinesisForwarder.forward("REMOVE", removed, null, scan.table(), scan.region(), scan.accountId());
}
}
- persistItemsForAccount(accountId, storageKey, items);
- totalDeleted += expiredKeys.size();
+ if (deletedForTable > 0) {
+ persistItemsForAccount(scan.accountId(), scan.storageKey(), items);
+ totalDeleted += deletedForTable;
+ }
}
if (totalDeleted > 0) {
LOG.infov("TTL sweeper removed {0} expired items", totalDeleted);
@@ -3071,8 +3105,13 @@ public class DynamoDbService implements ResourceProvider {
}
private ReentrantLock lockFor(String storageKey, String itemKey) {
+ return lockForScopedKey(scopedItemsKey(storageKey), itemKey);
+ }
+
+ // The sweeper has no request scope, so it locks with the raw account-scoped key instead of scopedItemsKey.
+ private ReentrantLock lockForScopedKey(String scopedKey, String itemKey) {
return itemLocks
- .computeIfAbsent(scopedItemsKey(storageKey), k -> new ConcurrentHashMap<>())
+ .computeIfAbsent(scopedKey, k -> new ConcurrentHashMap<>())
.computeIfAbsent(itemKey, k -> new ReentrantLock());
}
@@ -3096,6 +3135,16 @@ public class DynamoDbService implements ResourceProvider {
}
}
+ private <T> T withScopedItemLock(String scopedKey, String itemKey, Supplier<T> body) {
+ ReentrantLock lock = lockForScopedKey(scopedKey, itemKey);
+ lock.lock();
+ try {
+ return body.get();
+ } finally {
+ lock.unlock();
+ }
+ }
+
// AWS words a key rejection by the surface the key arrived on. A PutItem item body
// names the mismatched types, a Key argument and a BatchWriteItem entry report a
// schema mismatch instead. An empty key value is worded the same on every surface.
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
index cc0cf2afa3..64f9bedae7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
@@ -9,8 +9,10 @@ import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;
+import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
import io.github.hectorvent.floci.services.kinesis.KinesisService;
import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
@@ -198,4 +200,70 @@ class DynamoDbTtlForwardingTest {
assertNull(reloaded.getItem("TtlTable", key, REGION), "expired removal must be persisted");
}
}
+
+ // A write landing between scanExpiredItems() and deleteScannedItems() must not be lost.
+ @Test
+ void ttlSweep_preserves_item_updated_between_scan_and_delete() throws Exception {
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();
+ DynamoDbStreamService streamService = new DynamoDbStreamService(mapper, tableStore);
+ KinesisService kinesis = realKinesis();
+ KinesisStreamingForwarder forwarder = new KinesisStreamingForwarder(kinesis, mapper);
+ DynamoDbService svc = new DynamoDbService(
+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"),
+ streamService, forwarder);
+
+ seedTtlTable(svc, 2);
+ TableDefinition table = tableStore.get("us-east-1::TtlTable").orElseThrow();
+ StreamDescription sd = streamService.enableStream(
+ table.getTableName(), table.getTableArn(), "NEW_AND_OLD_IMAGES", REGION);
+ KinesisStream stream = kinesis.createStream("ttl-race-stream", 1, REGION);
+ table.getKinesisStreamingDestinations().add(new KinesisStreamingDestination(stream.getStreamArn()));
+
+ List<DynamoDbService.ExpiredTableScan> scans = svc.scanExpiredItems();
+ int candidateCount = scans.stream().mapToInt(scan -> scan.itemKeys().size()).sum();
+ assertEquals(2, candidateCount, "both expired rows must be scan candidates");
+
+ // Lands after the scan but before the delete step runs against that stale snapshot.
+ ObjectNode refreshed = mapper.createObjectNode();
+ refreshed.set("pk", stringAttr("row-0"));
+ refreshed.set("expireAt", numberAttr(Instant.now().getEpochSecond() + 3600));
+ refreshed.set("marker", stringAttr("updated-during-sweep"));
+ svc.putItem("TtlTable", refreshed, REGION);
+
+ svc.deleteScannedItems(scans);
+ assertTrue(forwarder.awaitIdle(Duration.ofSeconds(5)), "CDC forwards should drain promptly");
+
+ ObjectNode keyRow0 = mapper.createObjectNode();
+ keyRow0.set("pk", stringAttr("row-0"));
+ JsonNode row0 = svc.getItem("TtlTable", keyRow0, REGION);
+ assertNotNull(row0, "the concurrently updated item must survive the sweep");
+ assertEquals("updated-during-sweep", row0.get("marker").get("S").asText());
+
+ ObjectNode keyRow1 = mapper.createObjectNode();
+ keyRow1.set("pk", stringAttr("row-1"));
+ assertNull(svc.getItem("TtlTable", keyRow1, REGION), "the genuinely expired item must still be removed");
+
+ // The survivor's own update also lands on both destinations as a MODIFY; only the
+ // REMOVE side is what this test is about, so isolate that event before asserting on it.
+ String iterator = streamService.getShardIterator(
+ sd.getStreamArn(), DynamoDbStreamService.SHARD_ID, "TRIM_HORIZON", null);
+ DynamoDbStreamService.GetRecordsResult pulled = streamService.getRecords(iterator, 1000);
+ List<DynamoDbStreamRecord> removes = pulled.records().stream()
+ .filter(record -> "REMOVE".equals(record.getEventName()))
+ .toList();
+ assertEquals(1, removes.size(), "a skipped delete must not emit a stream REMOVE record");
+ assertEquals("row-1", removes.get(0).getKeys().get("pk").get("S").asText());
+
+ List<KinesisRecord> drained = drain(kinesis, "ttl-race-stream");
+ List<JsonNode> kinesisRemoves = new ArrayList<>();
+ for (KinesisRecord record : drained) {
+ JsonNode payload = mapper.readTree(record.getData());
+ if ("REMOVE".equals(payload.get("eventName").asText())) {
+ kinesisRemoves.add(payload);
+ }
+ }
+ assertEquals(1, kinesisRemoves.size(), "a skipped delete must not forward a Kinesis REMOVE record");
+ assertEquals("row-1", kinesisRemoves.get(0).get("dynamodb").get("Keys").get("pk").get("S").asText());
+ }
}