Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

refactor(dynamodb): scope runtime work to the selected backend (#4401)

Mario Toffiad658c75a413d
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index 0aa5ed41a9..cbf44c7670 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -59,6 +59,17 @@ Redshift zero-ETL integrations can consume these stream records directly. See th
[Redshift DynamoDB zero-ETL](redshift.md#dynamodb-zero-etl) section for the supported target,
landing table, checkpoint, and retry behavior.
+Stream records are held in memory. `POST /_floci/state/reset` removes every stream along with the
+tables, so `ListStreams` and `DescribeStream` no longer return the streams of tables created before
+the reset.
+
+## Time to Live
+
+With TTL enabled, a sweep runs every 60 seconds and deletes the items whose TTL attribute holds an
+epoch time in the past. Each deletion writes a `REMOVE` record to the table's stream and is
+forwarded to an active Kinesis streaming destination. Between sweeps, reads already leave expired
+items out.
+
## Configuration
| Variable | Default | Description |
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/Resettable.java b/src/main/java/io/github/hectorvent/floci/core/common/Resettable.java
index 7a23bd59fb..90960d33fe 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/Resettable.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/Resettable.java
@@ -7,6 +7,12 @@ package io.github.hectorvent.floci.core.common;
* controller never holds StorageFactory's monitor while invoking them.
*/
public interface Resettable {
+ /**
+ * Throw to refuse an emulator-wide reset. Runs on every service before anything is stopped or
+ * cleared, so a refusal leaves all state intact.
+ */
+ default void checkReset() {}
+
/** Stop accepting work and drain active writes before any storage is cleared. */
default void beforeReset() {}
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoController.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoController.java
index b8b4d63757..8f90b27a1d 100644
--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoController.java
+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoController.java
@@ -156,17 +156,21 @@ public class EmulatorInfoController {
}
private synchronized void performReset() {
- // Containers first: they are tracked independently of StorageBackend, so this can run
- // in any order relative to the storage wipe below, but stopping them here means a
- // client's reset actually reflects a clean slate instead of leaving Batch, CodeBuild,
- // or SageMaker containers running with no record of them left in the store.
- ContainerTeardowns.stopAll(containerTeardowns, LOG);
// Resolve live instances before taking any storage locks. Serialize resets so one reset
// cannot resume a publisher while another is still wiping its storage.
List<Resettable> services = new ArrayList<>();
for (Resettable service : resettables) {
services.add(service);
}
+ // Every service may refuse before the first destructive step, so a refusal changes nothing.
+ for (Resettable service : services) {
+ service.checkReset();
+ }
+ // Containers next: they are tracked independently of StorageBackend, so this can run
+ // in any order relative to the storage wipe below, but stopping them here means a
+ // client's reset actually reflects a clean slate instead of leaving Batch, CodeBuild,
+ // or SageMaker containers running with no record of them left in the store.
+ ContainerTeardowns.stopAll(containerTeardowns, LOG);
RuntimeException failure = null;
try {
for (Resettable service : services) {
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
index 91d1b46fe4..1d4b60363c 100644
--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
@@ -21,6 +21,7 @@ import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheCont
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;
import io.github.hectorvent.floci.services.elasticache.proxy.ElastiCacheProxyManager;
import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbRuntime;
import io.github.hectorvent.floci.services.lambda.DynamoDbStreamsEventSourcePoller;
import io.github.hectorvent.floci.services.lambda.KinesisEventSourcePoller;
import io.github.hectorvent.floci.services.lambda.SqsEventSourcePoller;
@@ -36,6 +37,7 @@ import io.github.hectorvent.floci.services.memorydb.container.MemoryDbContainerM
import io.github.hectorvent.floci.services.memorydb.proxy.MemoryDbProxyManager;
import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;
+import io.github.hectorvent.floci.services.redshift.RedshiftDynamoDbZeroEtlConsumer;
import io.github.hectorvent.floci.services.timestreaminfluxdb.TimestreamInfluxDbService;
import io.quarkus.runtime.Quarkus;
import io.quarkus.runtime.ShutdownDelayInitiatedEvent;
@@ -106,6 +108,8 @@ public class EmulatorLifecycle {
private final StepFunctionsService stepFunctionsService;
private final Instance<ContainerTeardown> containerTeardowns;
private final PersistentPathValidator persistentPathValidator;
+ private final DynamoDbRuntime dynamoDbRuntime;
+ private final RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer;
@Inject
public EmulatorLifecycle(StorageFactory storageFactory, ServiceRegistry serviceRegistry,
@@ -142,7 +146,9 @@ public class EmulatorLifecycle {
SchemaCreationWorker schemaCreationWorker,
StepFunctionsService stepFunctionsService,
Instance<ContainerTeardown> containerTeardowns,
- PersistentPathValidator persistentPathValidator) {
+ PersistentPathValidator persistentPathValidator,
+ DynamoDbRuntime dynamoDbRuntime,
+ RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer) {
this.storageFactory = storageFactory;
this.serviceRegistry = serviceRegistry;
this.config = config;
@@ -179,6 +185,8 @@ public class EmulatorLifecycle {
this.stepFunctionsService = stepFunctionsService;
this.containerTeardowns = containerTeardowns;
this.persistentPathValidator = persistentPathValidator;
+ this.dynamoDbRuntime = dynamoDbRuntime;
+ this.redshiftZeroEtlConsumer = redshiftZeroEtlConsumer;
}
void onStart(@Observes StartupEvent ignored) {
@@ -224,10 +232,13 @@ public class EmulatorLifecycle {
schemaCreationWorker.rehydrateSchemas();
stepFunctionsService.abortAbandonedExecutions();
+ // The selected DynamoDB backend is ready before any persisted stream consumer reads it.
+ dynamoDbRuntime.start();
sqsPoller.startPersistedPollers();
kinesisPoller.startPersistedPollers();
dynamodbStreamsPoller.startPersistedPollers();
pipesService.startPersistedPollers();
+ redshiftZeroEtlConsumer.startPersistedIntegrations();
rdsService.restorePersistedRuntime();
if (config.services().timestreamInfluxdb().enabled()) {
timestreamInfluxDbService.restorePersistedRuntime();
@@ -343,6 +354,9 @@ public class EmulatorLifecycle {
// SIGTERM grace window and trigger SIGKILL; if the flush ran last it would be skipped and
// in-memory (hybrid) data would be lost on an otherwise-graceful shutdown. shutdownAll()
// still runs at the end to stop the flush schedulers and capture any shutdown-time writes.
+ // DynamoDB stream consumers stop first, so the flush holds their final checkpoints.
+ runCleanup("DynamoDB Streams poller", dynamodbStreamsPoller::shutdown);
+ runCleanup("Redshift zero-ETL consumer", redshiftZeroEtlConsumer::shutdown);
runCleanup("storage flush", storageFactory::flushAll);
runCleanup("EC2 metadata server", () -> {
if (isMetadataServerNeeded()) {
@@ -372,6 +386,7 @@ public class EmulatorLifecycle {
// EC2 instances, in-flight build/job containers). Runs before shutdownAll() so any
// state written while stopping is captured by the final flush.
ContainerTeardowns.stopAll(containerTeardowns, LOG);
+ runCleanup("DynamoDB backend", dynamoDbRuntime::stop);
runCleanup("storage shutdown", storageFactory::shutdownAll);
LOG.info("=== AWS Local Emulator Stopped ===");
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntime.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntime.java
new file mode 100644
index 0000000000..44021473c0
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntime.java
@@ -0,0 +1,68 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.Resettable;
+import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbBackendLifecycle;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+
+/**
+ * Drives the selected DynamoDB backend through the emulator lifecycle: readiness at startup,
+ * reset preflight, quiescing and cleanup, and shutdown. Does nothing while DynamoDB is disabled.
+ */
+@ApplicationScoped
+public class DynamoDbRuntime implements Resettable {
+
+ private final DynamoDbBackendLifecycle backend;
+ private final EmulatorConfig config;
+
+ @Inject
+ public DynamoDbRuntime(DynamoDbBackendLifecycle backend, EmulatorConfig config) {
+ this.backend = backend;
+ this.config = config;
+ }
+
+ public void start() {
+ if (enabled()) {
+ backend.start();
+ }
+ }
+
+ public void stop() {
+ if (enabled()) {
+ backend.stop();
+ }
+ }
+
+ @Override
+ public void checkReset() {
+ if (enabled()) {
+ backend.checkReset();
+ }
+ }
+
+ @Override
+ public void beforeReset() {
+ if (enabled()) {
+ backend.beforeReset();
+ }
+ }
+
+ @Override
+ public void clear() {
+ if (enabled()) {
+ backend.reset();
+ }
+ }
+
+ @Override
+ public void afterReset() {
+ if (enabled()) {
+ backend.afterReset();
+ }
+ }
+
+ private boolean enabled() {
+ return config.services().dynamodb().enabled();
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 14ed108f3f..61b9951c30 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -227,6 +227,26 @@ public class DynamoDbService {
recoverInterruptedJobs();
}
+ /** Emulator reset: drops the in-process item cache, locks and idempotency tokens, and stream state. */
+ public void clearProcessState() {
+ itemsByTable.clear();
+ itemLocks.clear();
+ txIdempotency.clear();
+ if (streamService != null) {
+ streamService.clear();
+ }
+ if (kinesisForwarder != null) {
+ kinesisForwarder.clear();
+ }
+ }
+
+ /** Shutdown: ends Kinesis forwarding drains so none writes to Kinesis during the final storage flush. */
+ public void stopKinesisForwarding() {
+ if (kinesisForwarder != null) {
+ kinesisForwarder.clear();
+ }
+ }
+
private void loadPersistedItems() {
if (itemStore == null) return;
Map<String, TableDefinition> persistedTables = persistedTablesByScopedKey();
@@ -2687,7 +2707,8 @@ public class DynamoDbService {
}
record ExpiredTableScan(String rawKey, String accountId, String storageKey, String region,
- TableDefinition table, List<String> itemKeys) {}
+ TableDefinition table, List<String> itemKeys,
+ ConcurrentSkipListMap<String, JsonNode> items) {}
void deleteExpiredItems() {
deleteScannedItems(scanExpiredItems());
@@ -2726,7 +2747,7 @@ public class DynamoDbService {
String accountId = slash >= 0 ? rawKey.substring(0, slash) : null;
String storageKey = slash >= 0 ? rawKey.substring(slash + 1) : rawKey;
String region = storageKey.split("::", 2)[0];
- scans.add(new ExpiredTableScan(rawKey, accountId, storageKey, region, table, expiredKeys));
+ scans.add(new ExpiredTableScan(rawKey, accountId, storageKey, region, table, expiredKeys, items));
}
return scans;
}
@@ -2734,13 +2755,14 @@ public class DynamoDbService {
void deleteScannedItems(List<ExpiredTableScan> scans) {
int totalDeleted = 0;
for (ExpiredTableScan scan : scans) {
- ConcurrentSkipListMap<String, JsonNode> items = itemsByTable.get(scan.rawKey());
- if (items == null) {
- continue;
- }
+ ConcurrentSkipListMap<String, JsonNode> items = scan.items();
int deletedForTable = 0;
for (String itemKey : scan.itemKeys()) {
+ // A table deleted, or deleted and recreated, since the scan or during this sweep: its items are not this scan's.
+ if (itemsByTable.get(scan.rawKey()) != items) {
+ break;
+ }
JsonNode removed = withScopedItemLock(scan.rawKey(), itemKey, () -> {
JsonNode current = items.get(itemKey);
if (current == null || !isExpired(current, scan.table())) {
@@ -2762,7 +2784,15 @@ public class DynamoDbService {
}
}
if (deletedForTable > 0) {
- persistItemsForAccount(scan.accountId(), scan.storageKey(), items);
+ // A DeleteTable during the loop detached this map; writing it back would revive its items.
+ // Persisting under the entry's lock holds off DeleteTable, which removes the entry before
+ // it deletes the stored items.
+ itemsByTable.computeIfPresent(scan.rawKey(), (key, live) -> {
+ if (live == items) {
+ persistItemsForAccount(scan.accountId(), scan.storageKey(), items);
+ }
+ return live;
+ });
totalDeleted += deletedForTable;
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
index f5327c2311..95bbb67d58 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
@@ -146,6 +146,13 @@ public class DynamoDbStreamService {
}
}
+ /** Emulator reset: drops every stream and its records. Sequence numbers stay monotonic. */
+ public void clear() {
+ streams.clear();
+ records.clear();
+ streamRecordCounts.clear();
+ }
+
public void captureEvent(String eventName, JsonNode oldItem, JsonNode newItem,
TableDefinition table, String region) {
StreamDescription sd = streams.get(table.getTableArn());
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlService.java
index 018c2c8348..b9676a3ef0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlService.java
@@ -1,6 +1,5 @@
package io.github.hectorvent.floci.services.dynamodb;
-import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -17,6 +16,7 @@ public class DynamoDbTtlService {
private final DynamoDbService dynamoDbService;
private final ScheduledExecutorService scheduler;
+ private boolean paused;
@Inject
public DynamoDbTtlService(DynamoDbService dynamoDbService) {
@@ -28,14 +28,35 @@ public class DynamoDbTtlService {
});
}
- @PostConstruct
- void init() {
- scheduler.scheduleAtFixedRate(dynamoDbService::deleteExpiredItems, 60, 60, TimeUnit.SECONDS);
+ public void start() {
+ scheduler.scheduleAtFixedRate(this::sweep, 60, 60, TimeUnit.SECONDS);
LOG.infov("DynamoDB TTL sweeper scheduled (60s interval)");
}
+ synchronized void sweep() {
+ if (paused) {
+ return;
+ }
+ try {
+ dynamoDbService.deleteExpiredItems();
+ } catch (RuntimeException e) {
+ LOG.warnv(e, "DynamoDB TTL sweep failed; retrying on the next run");
+ }
+ }
+
+ /** Skips sweeps until {@link #resume()}, first waiting for a sweep already running to finish. */
+ public synchronized void pause() {
+ paused = true;
+ }
+
+ public synchronized void resume() {
+ paused = false;
+ }
+
+ /** Stops sweeping for good, first waiting for a sweep already running to finish. */
@PreDestroy
- void shutdown() {
+ public void stop() {
+ pause();
scheduler.shutdownNow();
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
index d5a71c2d4b..02aaf82f7f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
@@ -374,6 +374,12 @@ public class KinesisStreamingForwarder {
}
}
+ /** Emulator reset and shutdown: discard every destination's buffer, stop its drains, and drop its state. */
+ public void clear() {
+ states.values().forEach(this::invalidate);
+ states.clear();
+ }
+
@PreDestroy
public void shutdown() {
scheduler.shutdown();
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendLifecycle.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendLifecycle.java
new file mode 100644
index 0000000000..f7c5e33e3c
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendLifecycle.java
@@ -0,0 +1,23 @@
+package io.github.hectorvent.floci.services.dynamodb.backend;
+
+/** The selected engine's process lifecycle, driven by {@code DynamoDbRuntime}. */
+public interface DynamoDbBackendLifecycle {
+
+ /** Initializes the engine and its maintenance work; throws when it cannot serve requests. */
+ void start();
+
+ /** Throws to refuse an emulator-wide reset; runs before anything is stopped or cleared. */
+ void checkReset();
+
+ /** Quiesces maintenance work before Floci storage is cleared. */
+ void beforeReset();
+
+ /** Drops the engine's process state; runs after Floci storage was cleared. */
+ void reset();
+
+ /** Resumes maintenance work at the end of a reset; may run without a preceding {@link #beforeReset()}. */
+ void afterReset();
+
+ /** Stops maintenance work and closes clients. */
+ void stop();
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelector.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelector.java
index 4c18b42c72..76a5eb04ab 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelector.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelector.java
@@ -9,7 +9,7 @@ import jakarta.inject.Inject;
* The one immutable DynamoDB backend selection. Native is the only engine today; a configurable
* choice arrives with the alternative engine. {@link NativeDynamoDbBackend} is typed to its concrete
* class and {@link DynamoDbApiStreamReader} is not a bean, so these producers are the only
- * unqualified beans for the four seam interfaces.
+ * unqualified beans for the five seam interfaces.
*/
@ApplicationScoped
public class DynamoDbBackendSelector {
@@ -36,6 +36,11 @@ public class DynamoDbBackendSelector {
return nativeBackend;
}
+ @Produces
+ DynamoDbBackendLifecycle lifecycle() {
+ return nativeBackend;
+ }
+
/** Native stream history lives only in memory, so committed progress ends with the process. */
@Produces
DynamoDbStreamReader streamReader() {
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/NativeDynamoDbBackend.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/NativeDynamoDbBackend.java
index 230db5390a..71cd5998d5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/NativeDynamoDbBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/NativeDynamoDbBackend.java
@@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.resource.ExplorerResource;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlService;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbJsonHandler;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbStreamsJsonHandler;
import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
@@ -12,6 +14,7 @@ import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.quarkus.arc.ClientProxy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.inject.Typed;
import jakarta.inject.Inject;
@@ -27,23 +30,66 @@ import java.util.Optional;
*/
@ApplicationScoped
@Typed(NativeDynamoDbBackend.class)
-public class NativeDynamoDbBackend implements DynamoDbOperations, DynamoDbItemAccess, DynamoDbTableAccess {
+public class NativeDynamoDbBackend implements DynamoDbOperations, DynamoDbItemAccess, DynamoDbTableAccess,
+ DynamoDbBackendLifecycle {
private final NativeDynamoDbJsonHandler jsonHandler;
private final NativeDynamoDbStreamsJsonHandler streamsHandler;
private final DynamoDbService dynamoDbService;
+ private final DynamoDbStreamService streamService;
+ private final DynamoDbTtlService ttlService;
private final ObjectMapper objectMapper;
@Inject
public NativeDynamoDbBackend(NativeDynamoDbJsonHandler jsonHandler,
NativeDynamoDbStreamsJsonHandler streamsHandler,
- DynamoDbService dynamoDbService, ObjectMapper objectMapper) {
+ DynamoDbService dynamoDbService, DynamoDbStreamService streamService,
+ DynamoDbTtlService ttlService, ObjectMapper objectMapper) {
this.jsonHandler = jsonHandler;
this.streamsHandler = streamsHandler;
this.dynamoDbService = dynamoDbService;
+ this.streamService = streamService;
+ this.ttlService = ttlService;
this.objectMapper = objectMapper;
}
+ /**
+ * Resolves the engine beans so their constructors load persisted items, recover interrupted
+ * jobs and rebuild stream buffers now, before any consumer reads, then schedules the TTL sweep.
+ */
+ @Override
+ public void start() {
+ ClientProxy.unwrap(dynamoDbService);
+ ClientProxy.unwrap(streamService);
+ ttlService.start();
+ }
+
+ /** A native reset is always allowed. */
+ @Override
+ public void checkReset() {
+ }
+
+ @Override
+ public void beforeReset() {
+ ttlService.pause();
+ }
+
+ @Override
+ public void reset() {
+ dynamoDbService.clearProcessState();
+ }
+
+ @Override
+ public void afterReset() {
+ ttlService.resume();
+ }
+
+ @Override
+ public void stop() {
+ ttlService.stop();
+ dynamoDbService.stopKinesisForwarding();
+ }
+
@Override
public Reply execute(Call call) throws Exception {
String region = call.scope().region();
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index a9b311df9d..a22c52dc9e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -75,7 +75,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
final Set<String> stopped = ConcurrentHashMap.newKeySet();
/** {@code uuid:shardId} of shards a mapping read to their end; re-derived from committed progress. */
private final Set<String> finishedShards = ConcurrentHashMap.newKeySet();
- private volatile boolean resetting;
+ /** Set during a reset and after shutdown: no invocation starts and no checkpoint is saved. */
+ private volatile boolean quiesced;
private final ExecutorService pollExecutor = Executors.newCachedThreadPool(r -> {
Thread t = new Thread(r, "dynamodb-streams-esm-poller");
t.setDaemon(true);
@@ -185,8 +186,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
}
}
+ /**
+ * Stops polling for good. Taking the monitor drains an in-flight advanceCheckpoint save, and no
+ * checkpoint is saved after this returns. Idempotent, so it may run again from {@code @PreDestroy}.
+ */
@PreDestroy
- void shutdown() {
+ public synchronized void shutdown() {
+ quiesced = true;
pollExecutor.shutdownNow();
timerIds.values().forEach(vertx::cancelTimer);
timerIds.clear();
@@ -195,12 +201,12 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
@Override
public synchronized void beforeReset() {
// Taking the monitor drains an in-flight advanceCheckpoint save; it is released before the wipe.
- resetting = true;
+ quiesced = true;
}
@Override
public synchronized void afterReset() {
- resetting = false;
+ quiesced = pollExecutor.isShutdown();
}
public void clear() {
@@ -326,7 +332,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
boolean enabled = esmStore.getForAccount(esm.getAccountId(), esm.getUuid())
.map(EventSourceMapping::isEnabled)
.orElse(false);
- if (resetting || !enabled) {
+ if (quiesced || !enabled) {
return;
}
@@ -755,7 +761,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
}
private boolean isStopped(EventSourceMapping esm) {
- return resetting || stopped.contains(esm.getUuid()) || !exists(esm);
+ return quiesced || stopped.contains(esm.getUuid()) || !exists(esm);
}
private boolean exists(EventSourceMapping esm) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java
index dd214e8f97..6c4b61b6a0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbFacade;
import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbItemAccess.ScanPage;
import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbOperations.Scope;
@@ -16,11 +17,9 @@ import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
import io.github.hectorvent.floci.services.redshift.model.Integration;
import io.github.hectorvent.floci.services.redshiftdata.RedshiftZeroEtlWriter;
-import io.quarkus.runtime.StartupEvent;
import io.vertx.core.Vertx;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
-import jakarta.enterprise.event.Observes;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
@@ -33,9 +32,11 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
@ApplicationScoped
-public class RedshiftDynamoDbZeroEtlConsumer {
+public class RedshiftDynamoDbZeroEtlConsumer implements Resettable {
private static final Logger LOG = Logger.getLogger(RedshiftDynamoDbZeroEtlConsumer.class);
private static final int BATCH_SIZE = 100;
@@ -56,6 +57,11 @@ public class RedshiftDynamoDbZeroEtlConsumer {
thread.setDaemon(true);
return thread;
});
+ /** Polls hold the read lock; a reset takes the write lock to wait them out. */
+ private final ReentrantReadWriteLock quiesceLock = new ReentrantReadWriteLock();
+ private volatile boolean quiesced;
+ /** Completed resets: a poll writes only while none is in progress and none completed since its submission. */
+ private volatile long resets;
@Inject
public RedshiftDynamoDbZeroEtlConsumer(Vertx vertx,
@@ -92,10 +98,6 @@ public class RedshiftDynamoDbZeroEtlConsumer {
this.pollIntervalMs = pollIntervalMs;
}
- void onStart(@Observes StartupEvent event) {
- startPersistedIntegrations();
- }
-
/**
* Starts the persisted integrations, first discarding their stream progress when it lasts only
* for the process: native stream history is volatile, so a sequence number saved by a previous
@@ -116,8 +118,8 @@ public class RedshiftDynamoDbZeroEtlConsumer {
if (vertx == null || timerIds.containsKey(integration.getIntegrationArn())) {
return;
}
- long timerId = vertx.setPeriodic(pollIntervalMs, ignored -> pollAnd(integration));
- timerIds.put(integration.getIntegrationArn(), timerId);
+ timerIds.put(integration.getIntegrationArn(),
+ vertx.setPeriodic(pollIntervalMs, timerId -> pollAnd(integration, timerId)));
}
public void stopPolling(String integrationArn) {
@@ -130,6 +132,10 @@ public class RedshiftDynamoDbZeroEtlConsumer {
}
void pollOnce(Integration integration) {
+ unlessQuiesced(resets, () -> poll(integration));
+ }
+
+ private void poll(Integration integration) {
writer.createLandingTable(integration.getAccountId(), integration.getTargetClusterIdentifier(),
integration.getLandingTableName());
if (!integration.isBackfillCompleted()) {
@@ -261,7 +267,22 @@ public class RedshiftDynamoDbZeroEtlConsumer {
}
}
- public void reset() {
+ /**
+ * Waits for the poll in progress, then keeps every poll from writing until {@link #afterReset()},
+ * and a poll submitted before that from writing at all.
+ */
+ @Override
+ public void beforeReset() {
+ quiesceLock.writeLock().lock();
+ try {
+ quiesced = true;
+ } finally {
+ quiesceLock.writeLock().unlock();
+ }
+ }
+
+ @Override
+ public void clear() {
for (String integrationArn : timerIds.keySet()) {
stopPolling(integrationArn);
}
@@ -269,34 +290,86 @@ public class RedshiftDynamoDbZeroEtlConsumer {
finishedShards.clear();
}
+ /** Lets the polls submitted from now on write again. */
+ @Override
+ public void afterReset() {
+ quiesceLock.writeLock().lock();
+ try {
+ resets++;
+ quiesced = false;
+ } finally {
+ quiesceLock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Stops every integration for good, waiting up to five seconds for a poll in progress so the storage
+ * flush that follows holds its checkpoint. Idempotent, so it may run again from {@code @PreDestroy}.
+ */
@PreDestroy
- void onStop() {
- reset();
- pollExecutor.shutdownNow();
+ public void shutdown() {
+ clear();
+ pollExecutor.shutdown();
+ try {
+ if (!pollExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
+ pollExecutor.shutdownNow();
+ LOG.warnv("A zero-ETL poll was still running at shutdown, so its last checkpoint may not be flushed");
+ }
+ } catch (InterruptedException e) {
+ pollExecutor.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /** Runs the section unless a reset is in progress or one completed since {@code submittedAt} was read. */
+ private void unlessQuiesced(long submittedAt, Runnable section) {
+ quiesceLock.readLock().lock();
+ try {
+ if (!quiesced && resets == submittedAt) {
+ section.run();
+ }
+ } finally {
+ quiesceLock.readLock().unlock();
+ }
}
- private void pollAnd(Integration integration) {
+ /**
+ * Submits the tick's poll only while its timer is still registered, building the task first: {@code clear()}
+ * removes the timer before {@code afterReset()} counts the reset, both on the reset thread, so a task that
+ * read the new count sees its timer gone and one that read the old count is rejected by {@code unlessQuiesced}.
+ */
+ private void pollAnd(Integration integration, long timerId) {
+ Runnable task = pollTask(integration);
String integrationArn = integration.getIntegrationArn();
+ if (!Long.valueOf(timerId).equals(timerIds.get(integrationArn))) {
+ return;
+ }
if (activePolls.putIfAbsent(integrationArn, Boolean.TRUE) != null) {
return;
}
- pollExecutor.submit(() -> {
+ pollExecutor.submit(task);
+ }
+
+ /** The poll {@code pollAnd} submits, bound to the resets completed at its submission. */
+ Runnable pollTask(Integration integration) {
+ long submittedAt = resets;
+ return () -> {
try {
- pollSafely(integration);
+ pollSafely(integration, submittedAt);
} finally {
- activePolls.remove(integrationArn);
+ activePolls.remove(integration.getIntegrationArn());
}
- });
+ };
}
- private void pollSafely(Integration integration) {
+ private void pollSafely(Integration integration, long submittedAt) {
try {
- pollOnce(integration);
+ unlessQuiesced(submittedAt, () -> poll(integration));
} catch (Exception e) {
LOG.warnv(e, "Zero-ETL polling failed for integration {0}", integration.getIntegrationArn());
try {
- redshiftService.updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),
- integration.getShardSequenceNumbers(), false, e.getMessage());
+ unlessQuiesced(submittedAt, () -> redshiftService.updateIntegrationRuntime(integration.getAccountId(),
+ integration.getIntegrationArn(), integration.getShardSequenceNumbers(), false, e.getMessage()));
} catch (Exception updateError) {
LOG.warnv(updateError, "Could not persist zero-ETL failure for integration {0}",
integration.getIntegrationArn());
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/IamConditionContextResolverTest.java b/src/test/java/io/github/hectorvent/floci/core/common/IamConditionContextResolverTest.java
index 4d544422f9..59b55eabeb 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/IamConditionContextResolverTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/IamConditionContextResolverTest.java
@@ -56,7 +56,7 @@ class IamConditionContextResolverTest {
@BeforeEach
void setUp() {
dynamoDbService = mock(DynamoDbService.class);
- NativeDynamoDbBackend dynamoDbBackend = new NativeDynamoDbBackend(null, null, dynamoDbService, mapper);
+ NativeDynamoDbBackend dynamoDbBackend = new NativeDynamoDbBackend(null, null, dynamoDbService, null, null, mapper);
dynamoDbFacadeInstance = mock(Instance.class);
when(dynamoDbFacadeInstance.isResolvable()).thenReturn(true);
when(dynamoDbFacadeInstance.get()).thenReturn(new DynamoDbFacade(dynamoDbBackend, dynamoDbBackend,
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerIntegrationTest.java
index 4e850d0b7b..53ecfa5af5 100644
--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerIntegrationTest.java
@@ -1,15 +1,17 @@
package io.github.hectorvent.floci.lifecycle;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.JsonNode;
+import io.restassured.response.ValidatableResponse;
import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
-import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.UUID;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.*;
@@ -152,6 +154,35 @@ class EmulatorInfoControllerIntegrationTest {
.body("schemas[0]", equalTo("urn:ietf:params:scim:schemas:core:2.0:ServiceProviderConfig"));
}
+ @Test
+ void stateResetClearsDynamoDbStreams() {
+ String tableName = "reset-streams-" + UUID.randomUUID();
+ dynamoDb("DynamoDB_20120810.CreateTable", """
+ {"TableName": "%s",
+ "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}}
+ """.formatted(tableName))
+ .statusCode(200);
+ String listStreams = "{\"TableName\": \"" + tableName + "\"}";
+ dynamoDb("DynamoDBStreams_20120810.ListStreams", listStreams).body("Streams", hasSize(1));
+
+ given().when().post("/_floci/state/reset").then().statusCode(200);
+
+ dynamoDb("DynamoDBStreams_20120810.ListStreams", listStreams).statusCode(200).body("Streams", hasSize(0));
+ }
+
+ private static ValidatableResponse dynamoDb(String target, String body) {
+ return given()
+ .header("X-Amz-Target", target)
+ .contentType("application/x-amz-json-1.0")
+ .body(body)
+ .when()
+ .post("/")
+ .then();
+ }
+
@Test
void stateReset_clearsDatabaseState() {
// 1. Put SSM parameter
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerTest.java
index 1023f6964f..5fdcba6e69 100644
--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerTest.java
@@ -18,12 +18,16 @@ import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
@@ -148,4 +152,21 @@ class EmulatorInfoControllerTest {
verify(resettable, never()).clear();
verify(later, never()).clear();
}
+
+ @Test
+ void resetRefusedByPreflightChangesNothing() {
+ Resettable refusing = mock(Resettable.class);
+ IllegalStateException refusal = new IllegalStateException("backend refuses reset");
+ doThrow(refusal).when(refusing).checkReset();
+ lenient().when(containerTeardowns.iterator()).thenReturn(List.of(sageMakerTeardown).iterator());
+ when(resettables.iterator()).thenReturn(List.of(resettable, refusing).iterator());
+
+ assertSame(refusal, assertThrows(IllegalStateException.class, controller::reset));
+
+ verify(resettable).checkReset();
+ verify(refusing).checkReset();
+ verifyNoMoreInteractions(resettable, refusing);
+ verify(sageMakerTeardown, never()).stopManagedContainers();
+ verifyNoInteractions(storageFactory);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
index bdb65112ff..61b0c6d738 100644
--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;
import io.github.hectorvent.floci.lifecycle.inithook.InitializationHooksRunner;
import io.github.hectorvent.floci.services.appsync.graphql.SchemaCreationWorker;
import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbRuntime;
import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;
import io.github.hectorvent.floci.services.ecs.container.EcsTaskRoleCredentialsServer;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
@@ -28,6 +29,7 @@ import io.github.hectorvent.floci.services.pipes.PipesService;
import io.github.hectorvent.floci.services.rds.RdsService;
import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;
+import io.github.hectorvent.floci.services.redshift.RedshiftDynamoDbZeroEtlConsumer;
import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService;
import io.github.hectorvent.floci.services.timestreaminfluxdb.TimestreamInfluxDbService;
import io.quarkus.runtime.Quarkus;
@@ -113,6 +115,8 @@ class EmulatorLifecycleTest {
@Mock private SchemaCreationWorker schemaCreationWorker;
@Mock private StepFunctionsService stepFunctionsService;
@Mock private Instance<ContainerTeardown> containerTeardowns;
+ @Mock private DynamoDbRuntime dynamoDbRuntime;
+ @Mock private RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer;
private EmulatorLifecycle emulatorLifecycle;
@@ -151,7 +155,8 @@ class EmulatorLifecycleTest {
elbV2Service, elbClassicService,
initializationHooksRunner, sqsPoller, kinesisPoller, dynamodbStreamsPoller,
pipesService, ec2MetadataServer, ecsTaskRoleCredentialsServer, ecrRegistryManager, flociUiManager, initLifecycleState,
- schemaCreationWorker, stepFunctionsService, containerTeardowns, persistentPathValidator);
+ schemaCreationWorker, stepFunctionsService, containerTeardowns, persistentPathValidator,
+ dynamoDbRuntime, redshiftZeroEtlConsumer);
Mockito.lenient().when(containerTeardowns.iterator())
.thenReturn(java.util.Collections.emptyIterator());
}
@@ -181,6 +186,46 @@ class EmulatorLifecycleTest {
inOrder.verify(rdsService).restorePersistedRuntime();
}
+ @Test
+ void startsTheDynamoDbBackendAfterStorageLoadAndBeforeItsStreamConsumers() {
+ stubStorageConfig();
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+
+ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+
+ InOrder inOrder = Mockito.inOrder(storageFactory, dynamoDbRuntime, dynamodbStreamsPoller, pipesService,
+ redshiftZeroEtlConsumer);
+ inOrder.verify(storageFactory).loadAll();
+ inOrder.verify(dynamoDbRuntime).start();
+ inOrder.verify(dynamodbStreamsPoller).startPersistedPollers();
+ inOrder.verify(pipesService).startPersistedPollers();
+ inOrder.verify(redshiftZeroEtlConsumer).startPersistedIntegrations();
+ }
+
+ @Test
+ void stopsDynamoDbStreamConsumersBeforeTheFlushAndTheBackendBeforeStorageShutdown() {
+ emulatorLifecycle.onStop(Mockito.mock(ShutdownEvent.class));
+
+ InOrder inOrder = Mockito.inOrder(dynamodbStreamsPoller, redshiftZeroEtlConsumer, storageFactory,
+ flociUiManager, dynamoDbRuntime);
+ inOrder.verify(dynamodbStreamsPoller).shutdown();
+ inOrder.verify(redshiftZeroEtlConsumer).shutdown();
+ inOrder.verify(storageFactory).flushAll();
+ inOrder.verify(flociUiManager).shutdown();
+ inOrder.verify(dynamoDbRuntime).stop();
+ inOrder.verify(storageFactory).shutdownAll();
+ }
+
+ @Test
+ void stopsTheZeroEtlConsumerWhenTheStreamsPollerShutdownFails() {
+ doThrow(new IllegalStateException("poller shutdown failed")).when(dynamodbStreamsPoller).shutdown();
+
+ emulatorLifecycle.onStop(Mockito.mock(ShutdownEvent.class));
+
+ verify(redshiftZeroEtlConsumer).shutdown();
+ }
+
@Test
@DisplayName("Should restore ELBv2 persisted runtime after loading storage when elbv2 is enabled")
void shouldRestoreElbV2PersistedRuntimeAfterStorageLoad() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CfnProvisionerFixture.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CfnProvisionerFixture.java
index 9ee2e15b0c..c20f2461e0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CfnProvisionerFixture.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CfnProvisionerFixture.java
@@ -268,7 +268,7 @@ final class CfnProvisionerFixture {
discovered.add(new SnsCfnProvisioner(snsService));
}
if (dynamoDbService != null) {
- NativeDynamoDbBackend backend = new NativeDynamoDbBackend(null, null, dynamoDbService, objectMapper);
+ NativeDynamoDbBackend backend = new NativeDynamoDbBackend(null, null, dynamoDbService, null, null, objectMapper);
discovered.add(new DynamoDbCfnProvisioner(new DynamoDbFacade(backend, backend,
new RegionResolver("us-east-1", "000000000000"))));
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/DynamoDbCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/DynamoDbCfnProvisionerTest.java
index 1d0b88d0fa..0059ec46a7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/DynamoDbCfnProvisionerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/DynamoDbCfnProvisionerTest.java
@@ -46,7 +46,7 @@ class DynamoDbCfnProvisionerTest {
private final DynamoDbService dynamoDb = mock(DynamoDbService.class);
private final ObjectMapper mapper = new ObjectMapper();
- private final NativeDynamoDbBackend backend = new NativeDynamoDbBackend(null, null, dynamoDb, mapper);
+ private final NativeDynamoDbBackend backend = new NativeDynamoDbBackend(null, null, dynamoDb, null, null, mapper);
private final DynamoDbCfnProvisioner provisioner = new DynamoDbCfnProvisioner(
new DynamoDbFacade(backend, backend, new RegionResolver("us-east-1", "000000000000")));
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntimeTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntimeTest.java
new file mode 100644
index 0000000000..d0696e49b1
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntimeTest.java
@@ -0,0 +1,55 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbBackendLifecycle;
+import org.junit.jupiter.api.Test;
+import org.mockito.InOrder;
+
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+class DynamoDbRuntimeTest {
+
+ private final DynamoDbBackendLifecycle backend = mock(DynamoDbBackendLifecycle.class);
+ private final EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ private final DynamoDbRuntime runtime = new DynamoDbRuntime(backend, config);
+
+ @Test
+ void drivesTheSelectedBackendLifecycle() {
+ when(config.services().dynamodb().enabled()).thenReturn(true);
+
+ runtime.start();
+ runtime.checkReset();
+ runtime.beforeReset();
+ runtime.clear();
+ runtime.afterReset();
+ runtime.stop();
+
+ InOrder order = inOrder(backend);
+ order.verify(backend).start();
+ order.verify(backend).checkReset();
+ order.verify(backend).beforeReset();
+ order.verify(backend).reset();
+ order.verify(backend).afterReset();
+ order.verify(backend).stop();
+ verifyNoMoreInteractions(backend);
+ }
+
+ @Test
+ void leavesTheBackendAloneWhenDynamoDbIsDisabled() {
+ when(config.services().dynamodb().enabled()).thenReturn(false);
+
+ runtime.start();
+ runtime.checkReset();
+ runtime.beforeReset();
+ runtime.clear();
+ runtime.afterReset();
+ runtime.stop();
+
+ verifyNoInteractions(backend);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
index 64f9bedae7..7de9fc8cfb 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
@@ -25,11 +25,17 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@@ -201,6 +207,127 @@ class DynamoDbTtlForwardingTest {
}
}
+ @Test
+ void ttlSweepDoesNotPersistItemsOfATableDeletedMidSweep() {
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();
+ DynamoDbStreamService streamService = mock(DynamoDbStreamService.class);
+ DynamoDbService svc = new DynamoDbService(
+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"), streamService, null);
+ seedTtlTable(svc, 1);
+ doAnswer(invocation -> {
+ svc.deleteTable("TtlTable", REGION);
+ return null;
+ }).when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any());
+
+ svc.deleteExpiredItems();
+
+ assertTrue(itemStore.get("us-east-1::TtlTable").isEmpty(),
+ "the sweep must not write items back for a table deleted while it ran");
+ }
+
+ @Test
+ void aSweepDoesNotDeleteFromATableRecreatedAfterItsScan() {
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();
+ DynamoDbService svc = new DynamoDbService(
+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"));
+ long past = Instant.now().getEpochSecond() - 3600;
+ svc.createTable("TtlTable",
+ List.of(new KeySchemaElement("pk", "HASH")),
+ List.of(new AttributeDefinition("pk", "S")),
+ 5L, 5L, REGION);
+ svc.updateTimeToLive("TtlTable", "expireAt", true, REGION);
+ ObjectNode expired = mapper.createObjectNode();
+ expired.set("pk", stringAttr("row-0"));
+ expired.set("expireAt", numberAttr(past));
+ svc.putItem("TtlTable", expired, REGION);
+ List<DynamoDbService.ExpiredTableScan> scans = svc.scanExpiredItems();
+ assertEquals(1, scans.stream().mapToInt(scan -> scan.itemKeys().size()).sum(),
+ "the expired row must be a scan candidate");
+
+ svc.deleteTable("TtlTable", REGION);
+ svc.createTable("TtlTable",
+ List.of(new KeySchemaElement("pk", "HASH")),
+ List.of(new AttributeDefinition("pk", "S")),
+ 5L, 5L, REGION);
+ ObjectNode recreated = mapper.createObjectNode();
+ recreated.set("pk", stringAttr("row-0"));
+ recreated.set("expireAt", numberAttr(past));
+ svc.putItem("TtlTable", recreated, REGION);
+ svc.deleteScannedItems(scans);
+
+ ObjectNode key = mapper.createObjectNode();
+ key.set("pk", stringAttr("row-0"));
+ assertNotNull(svc.getItem("TtlTable", key, REGION),
+ "a scan of the deleted table must not delete the recreated table's item");
+ }
+
+ @Test
+ void aTableDeletedDuringASweepEmitsNoFurtherRemoves() {
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();
+ DynamoDbStreamService streamService = mock(DynamoDbStreamService.class);
+ DynamoDbService svc = new DynamoDbService(
+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"), streamService, null);
+ seedTtlTable(svc, 3);
+ List<DynamoDbService.ExpiredTableScan> scans = svc.scanExpiredItems();
+ assertEquals(3, scans.stream().mapToInt(scan -> scan.itemKeys().size()).sum(),
+ "every expired row must be a scan candidate");
+ doAnswer(invocation -> {
+ svc.deleteTable("TtlTable", REGION);
+ return null;
+ }).doNothing().when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any());
+
+ svc.deleteScannedItems(scans);
+
+ verify(streamService, times(1)).captureEvent(eq("REMOVE"), any(), any(), any(), any());
+ }
+
+ @Test
+ void deleteTableWaitsForASweepPersistingTheTable() throws Exception {
+ CountDownLatch persisting = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>() {
+ @Override
+ public void put(String key, Map<String, JsonNode> value) {
+ if ("ttl-sweep".equals(Thread.currentThread().getName())) {
+ persisting.countDown();
+ try {
+ release.await(10, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ super.put(key, value);
+ }
+ };
+ DynamoDbService svc = new DynamoDbService(
+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"));
+ seedTtlTable(svc, 1);
+ Thread sweep = new Thread(svc::deleteExpiredItems, "ttl-sweep");
+ sweep.start();
+ assertTrue(persisting.await(10, TimeUnit.SECONDS));
+
+ Thread delete = new Thread(() -> svc.deleteTable("TtlTable", REGION));
+ delete.start();
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (delete.getState() != Thread.State.BLOCKED) {
+ if (System.nanoTime() > deadline) {
+ fail("DeleteTable did not wait for the sweep persisting the table");
+ }
+ Thread.onSpinWait();
+ }
+ release.countDown();
+ sweep.join(10_000);
+ delete.join(10_000);
+
+ assertEquals(Thread.State.TERMINATED, delete.getState());
+ assertTrue(itemStore.get("us-east-1::TtlTable").isEmpty(),
+ "DeleteTable must delete the stored items after the sweep wrote them");
+ }
+
// A write landing between scanExpiredItems() and deleteScannedItems() must not be lost.
@Test
void ttlSweep_preserves_item_updated_between_scan_and_delete() throws Exception {
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlServiceTest.java
new file mode 100644
index 0000000000..49f9917f3e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlServiceTest.java
@@ -0,0 +1,92 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+class DynamoDbTtlServiceTest {
+
+ @Test
+ void aFailingSweepDoesNotEscapeTheScheduledTask() {
+ DynamoDbService dynamoDbService = mock(DynamoDbService.class);
+ doThrow(new IllegalStateException("sweep failed")).when(dynamoDbService).deleteExpiredItems();
+ DynamoDbTtlService ttlService = new DynamoDbTtlService(dynamoDbService);
+
+ assertDoesNotThrow(ttlService::sweep);
+ }
+
+ @Test
+ void aPausedSweepDeletesNothingUntilResumed() {
+ DynamoDbService dynamoDbService = mock(DynamoDbService.class);
+ DynamoDbTtlService ttlService = new DynamoDbTtlService(dynamoDbService);
+
+ ttlService.pause();
+ ttlService.sweep();
+ verify(dynamoDbService, never()).deleteExpiredItems();
+
+ ttlService.resume();
+ ttlService.sweep();
+ verify(dynamoDbService).deleteExpiredItems();
+ }
+
+ @Test
+ void pauseWaitsForTheSweepInProgress() throws Exception {
+ assertWaitsForTheSweepInProgress(DynamoDbTtlService::pause);
+ }
+
+ @Test
+ void stopWaitsForASweepInProgress() throws Exception {
+ assertWaitsForTheSweepInProgress(DynamoDbTtlService::stop);
+ }
+
+ /** Also asserts that no sweep deletes anything after {@code call} returns. */
+ private static void assertWaitsForTheSweepInProgress(Consumer<DynamoDbTtlService> call) throws Exception {
+ DynamoDbService dynamoDbService = mock(DynamoDbService.class);
+ CountDownLatch sweeping = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ sweeping.countDown();
+ release.await(10, TimeUnit.SECONDS);
+ return null;
+ }).when(dynamoDbService).deleteExpiredItems();
+ DynamoDbTtlService ttlService = new DynamoDbTtlService(dynamoDbService);
+ Thread sweep = new Thread(ttlService::sweep);
+ sweep.start();
+ assertTrue(sweeping.await(10, TimeUnit.SECONDS));
+
+ Thread caller = new Thread(() -> call.accept(ttlService));
+ caller.start();
+ awaitBlocked(caller);
+ release.countDown();
+ caller.join(10_000);
+ sweep.join(10_000);
+
+ assertEquals(Thread.State.TERMINATED, caller.getState());
+ assertEquals(Thread.State.TERMINATED, sweep.getState());
+ ttlService.sweep();
+ verify(dynamoDbService).deleteExpiredItems();
+ }
+
+ /** Observes the calling thread reach the sweep's monitor instead of sleeping to hope for it. */
+ private static void awaitBlocked(Thread thread) {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (thread.getState() != Thread.State.BLOCKED) {
+ if (System.nanoTime() > deadline) {
+ fail("the call returned or never reached the sweep's monitor while a sweep was running");
+ }
+ Thread.onSpinWait();
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendResetIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendResetIntegrationTest.java
new file mode 100644
index 0000000000..64b9dffea8
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendResetIntegrationTest.java
@@ -0,0 +1,72 @@
+package io.github.hectorvent.floci.services.dynamodb.backend;
+
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.arc.Arc;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import io.restassured.response.ValidatableResponse;
+import jakarta.enterprise.context.ApplicationScoped;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.util.Set;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+/** The emulator lifecycle drives only the selected DynamoDB backend, here one that refuses resets. */
+@QuarkusTest
+@TestProfile(DynamoDbBackendResetIntegrationTest.RefusingBackendProfile.class)
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class DynamoDbBackendResetIntegrationTest {
+
+ public static final class RefusingBackendProfile implements QuarkusTestProfile {
+ @Override
+ public Set<Class<?>> getEnabledAlternatives() {
+ return Set.of(RefusingDynamoDbBackendLifecycle.class);
+ }
+ }
+
+ private static final String PARAMETER = "/dynamodb-backend-reset/survives-a-refused-reset";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void startupInitializesNoNativeDatabaseState() {
+ assertFalse(Arc.container().getActiveContext(ApplicationScoped.class).getState().getContextualInstances()
+ .keySet().stream().anyMatch(bean -> bean.getBeanClass() == DynamoDbService.class));
+ }
+
+ @Test
+ @Order(2)
+ void aRefusedResetClearsNothing() {
+ ssm("PutParameter", "{\"Name\": \"" + PARAMETER + "\", \"Value\": \"kept\", \"Type\": \"String\"}")
+ .statusCode(200);
+
+ given().when().post("/_floci/state/reset").then().statusCode(409);
+
+ ssm("GetParameter", "{\"Name\": \"" + PARAMETER + "\"}")
+ .statusCode(200)
+ .body("Parameter.Value", equalTo("kept"));
+ }
+
+ private static ValidatableResponse ssm(String action, String body) {
+ return given()
+ .header("X-Amz-Target", "AmazonSSM." + action)
+ .contentType("application/x-amz-json-1.1")
+ .body(body)
+ .when()
+ .post("/")
+ .then();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelectionTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelectionTest.java
index 712518ff15..c4dfdfbe03 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelectionTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelectionTest.java
@@ -54,6 +54,9 @@ class DynamoDbBackendSelectionTest {
@Inject
DynamoDbStreamReader streamReader;
+ @Inject
+ DynamoDbBackendLifecycle lifecycle;
+
// Collected exactly as ResourceExplorer2Service collects its providers.
@Inject
Instance<ResourceProvider> providers;
@@ -68,6 +71,13 @@ class DynamoDbBackendSelectionTest {
assertInstanceOf(NativeDynamoDbBackend.class, backend);
assertSame(backend, ClientProxy.unwrap(items));
assertSame(backend, ClientProxy.unwrap(tables));
+ assertSame(backend, ClientProxy.unwrap(lifecycle));
+ }
+
+ @Test
+ void startupSchedulesTheTtlSweep() {
+ assertTrue(Thread.getAllStackTraces().keySet().stream()
+ .anyMatch(thread -> thread.isAlive() && "dynamodb-ttl-sweeper".equals(thread.getName())));
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbNativeBackendTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbNativeBackendTest.java
index 956a9a7f2b..53c209449c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbNativeBackendTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbNativeBackendTest.java
@@ -7,11 +7,13 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamsJsonHandler;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlService;
import io.github.hectorvent.floci.services.dynamodb.KinesisStreamingForwarder;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbJsonHandler;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbStreamsJsonHandler;
@@ -26,6 +28,7 @@ import io.github.hectorvent.floci.services.kinesis.KinesisService;
import jakarta.ws.rs.core.Response;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.InOrder;
import java.util.ArrayList;
import java.util.List;
@@ -39,7 +42,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/** The native engine behind the DynamoDB seam, and the thin public handlers in front of it. */
@@ -51,20 +56,26 @@ class DynamoDbNativeBackendTest {
private static final String TABLE = "orders";
private final ObjectMapper mapper = new ObjectMapper();
+ private final List<StorageBackend<?, ?>> stores = new ArrayList<>();
+ private final KinesisStreamingForwarder forwarder = mock(KinesisStreamingForwarder.class);
+ private final DynamoDbTtlService ttl = mock(DynamoDbTtlService.class);
private NativeDynamoDbBackend backend;
@BeforeEach
void setUp() {
StorageFactory storageFactory = mock(StorageFactory.class);
- when(storageFactory.create(anyString(), anyString(), any()))
- .thenAnswer(invocation -> AccountAwareStorageBackend.inMemory(ACCOUNT));
+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(invocation -> {
+ AccountAwareStorageBackend<Object> store = AccountAwareStorageBackend.inMemory(ACCOUNT);
+ stores.add(store);
+ return store;
+ });
DynamoDbStreamService streams = new DynamoDbStreamService(mapper, storageFactory);
DynamoDbService service = new DynamoDbService(storageFactory, new RegionResolver(REGION, ACCOUNT), streams,
- mock(KinesisStreamingForwarder.class), null, mapper, mock(EmulatorConfig.class, RETURNS_DEEP_STUBS));
+ forwarder, null, mapper, mock(EmulatorConfig.class, RETURNS_DEEP_STUBS));
NativeDynamoDbTableService tables = new NativeDynamoDbTableService(service, streams, mock(KinesisService.class));
backend = new NativeDynamoDbBackend(new NativeDynamoDbJsonHandler(service, tables, mapper),
new NativeDynamoDbStreamsJsonHandler(streams, service, new RegionResolver(REGION, ACCOUNT), mapper),
- service, mapper);
+ service, streams, ttl, mapper);
}
private Reply call(Api api, String action, JsonNode body) throws Exception {
@@ -84,6 +95,52 @@ class DynamoDbNativeBackendTest {
return mapper.readTree(text);
}
+ @Test
+ void lifecycleDrivesTheTtlSweep() {
+ backend.start();
+ verify(ttl).start();
+
+ backend.beforeReset();
+ verify(ttl).pause();
+
+ backend.afterReset();
+ verify(ttl).resume();
+
+ backend.stop();
+ verify(ttl).stop();
+ }
+
+ @Test
+ void stopEndsKinesisForwardingAfterTheTtlSweep() {
+ backend.stop();
+
+ InOrder order = inOrder(ttl, forwarder);
+ order.verify(ttl).stop();
+ order.verify(forwarder).clear();
+ }
+
+ @Test
+ void resetDropsTheEngineProcessState() throws Exception {
+ createTable(",\"StreamSpecification\":{\"StreamEnabled\":true,\"StreamViewType\":\"KEYS_ONLY\"}");
+ JsonNode transact = json("""
+ {"ClientRequestToken":"token-1",
+ "TransactItems":[{"Put":{"TableName":"%s","Item":{"id":{"S":"a"}}}}]}
+ """.formatted(TABLE));
+ call(Api.DYNAMODB, "TransactWriteItems", transact);
+
+ // An emulator reset wipes Floci storage before it resets the engine.
+ stores.forEach(StorageBackend::clear);
+ backend.reset();
+ createTable("");
+ call(Api.DYNAMODB, "TransactWriteItems", transact);
+
+ Reply streams = call(Api.DYNAMODB_STREAMS, "ListStreams", mapper.createObjectNode());
+ assertEquals(0, streams.body().path("Streams").size());
+ JsonNode key = json("{\"id\":{\"S\":\"a\"}}");
+ assertEquals(key, backend.getItem(SCOPE, TABLE, key), "the reused token writes again");
+ verify(forwarder).clear();
+ }
+
@Test
void rawDeleteTableHonoursDeletionProtectionWhileTypedDeleteDoesNot() throws Exception {
createTable(",\"DeletionProtectionEnabled\":true");
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/RefusingDynamoDbBackendLifecycle.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/RefusingDynamoDbBackendLifecycle.java
new file mode 100644
index 0000000000..902333ed44
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/RefusingDynamoDbBackendLifecycle.java
@@ -0,0 +1,39 @@
+package io.github.hectorvent.floci.services.dynamodb.backend;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Alternative;
+
+/** A selected backend that starts nothing and refuses every emulator reset. */
+@Alternative
+@ApplicationScoped
+public class RefusingDynamoDbBackendLifecycle implements DynamoDbBackendLifecycle {
+
+ @Override
+ public void start() {
+ }
+
+ @Override
+ public void checkReset() {
+ throw new AwsException("ResourceInUseException", "The DynamoDB backend refuses an emulator reset", 409);
+ }
+
+ @Override
+ public void beforeReset() {
+ throw new IllegalStateException("beforeReset ran after a refused preflight");
+ }
+
+ @Override
+ public void reset() {
+ throw new IllegalStateException("reset ran after a refused preflight");
+ }
+
+ @Override
+ public void afterReset() {
+ throw new IllegalStateException("afterReset ran after a refused preflight");
+ }
+
+ @Override
+ public void stop() {
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
index 9100047e08..dd5aa083a0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
@@ -1955,6 +1955,50 @@ class DynamoDbStreamsEventSourcePollerTest {
verify(store).saveForAccount(eq(ACCOUNT_ID), any());
}
+ @Test
+ void checkpointIsNotSavedAfterShutdown() throws Exception {
+ stubStream("s1");
+ EventSourceMapping esm = filterEsm();
+ EsmStore store = mock(EsmStore.class);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);
+ AtomicBoolean shutdownDuringInvoke = new AtomicBoolean(true);
+ List<List<String>> invocations = recordInvocations(seqs -> {
+ if (shutdownDuringInvoke.getAndSet(false)) {
+ p.shutdown();
+ }
+ return new InvokeResult();
+ });
+
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1")), invocations);
+ assertNull(checkpoint(esm));
+ verify(store, never()).saveForAccount(anyString(), any());
+ }
+
+ @Test
+ void aResetEndingAfterShutdownKeepsThePollerQuiesced() throws Exception {
+ stubStream("s1");
+ EventSourceMapping esm = filterEsm();
+ EsmStore store = mock(EsmStore.class);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);
+ AtomicBoolean resetAcrossShutdownDuringInvoke = new AtomicBoolean(true);
+ List<List<String>> invocations = recordInvocations(seqs -> {
+ if (resetAcrossShutdownDuringInvoke.getAndSet(false)) {
+ p.beforeReset();
+ p.shutdown();
+ p.afterReset();
+ }
+ return new InvokeResult();
+ });
+
+ pollOnce(p, esm);
+
+ assertEquals(List.of(List.of("s1")), invocations);
+ assertNull(checkpoint(esm));
+ verify(store, never()).saveForAccount(anyString(), any());
+ }
+
@Test
void pollDuringAResetDoesNotInvokeUntilTheResetEnds() throws Exception {
stubStream("s1");
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java
index b4bd501c06..272d1d0356 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java
@@ -3,9 +3,11 @@ package io.github.hectorvent.floci.services.redshift;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbFacade;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;
import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;
@@ -17,6 +19,8 @@ import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.redshift.model.Integration;
import io.github.hectorvent.floci.services.redshiftdata.RedshiftZeroEtlWriter;
import io.quarkus.test.junit.QuarkusTest;
+import io.vertx.core.Handler;
+import io.vertx.core.Vertx;
import jakarta.inject.Inject;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
@@ -27,19 +31,27 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@QuarkusTest
@@ -260,6 +272,190 @@ class RedshiftDynamoDbZeroEtlConsumerTest {
return new DynamoDbStreamReader.Record(sequenceNumber, awsRecord);
}
+ @Test
+ void anEmulatorResetStopsEveryIntegrationTimer() {
+ Vertx vertx = mock(Vertx.class);
+ when(vertx.setPeriodic(anyLong(), any())).thenReturn(7L);
+ RedshiftDynamoDbZeroEtlConsumer consumer = new RedshiftDynamoDbZeroEtlConsumer(vertx,
+ new FakeStreamReader(CheckpointLifetime.PROCESS), dynamoDb, mock(RedshiftService.class),
+ mock(RedshiftZeroEtlWriter.class), MAPPER, mock(EmulatorConfig.class, RETURNS_DEEP_STUBS));
+ consumer.startPolling(streamingIntegration());
+ Resettable resettable = consumer;
+
+ resettable.clear();
+
+ verify(vertx).cancelTimer(7L);
+ }
+
+ @Test
+ void anEmulatorResetWaitsForTheWriteInProgressAndWritesNothingUntilItEnds() throws Exception {
+ FakeStreamReader streams = new FakeStreamReader(CheckpointLifetime.PROCESS);
+ RedshiftZeroEtlWriter writer = mock(RedshiftZeroEtlWriter.class);
+ CountDownLatch writing = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ writing.countDown();
+ release.await(10, TimeUnit.SECONDS);
+ return null;
+ }).doNothing().when(writer).writeBatch(any(), any(), any(), any());
+ Integration integration = streamingIntegration();
+ List<DynamoDbStreamReader.Record> shard = streams.shard(SHARD_ID, null);
+ shard.add(streamRecord("s1"));
+ RedshiftDynamoDbZeroEtlConsumer consumer =
+ new RedshiftDynamoDbZeroEtlConsumer(streams, dynamoDb, mock(RedshiftService.class), writer);
+ Thread poll = new Thread(() -> consumer.pollOnce(integration));
+ poll.start();
+ assertTrue(writing.await(10, TimeUnit.SECONDS));
+
+ Thread reset = new Thread(consumer::beforeReset);
+ reset.start();
+ awaitWaiting(reset);
+ release.countDown();
+ reset.join(10_000);
+ poll.join(10_000);
+ assertEquals(Thread.State.TERMINATED, reset.getState());
+
+ DynamoDbStreamReader.Record second = streamRecord("s2");
+ shard.add(second);
+ consumer.pollOnce(integration);
+ verify(writer, times(1)).createLandingTable(any(), any(), any());
+ verify(writer, times(1)).writeBatch(any(), any(), any(), any());
+
+ consumer.afterReset();
+ consumer.pollOnce(integration);
+ verify(writer).writeBatch(integration.getAccountId(), "warehouse", "floci_zetl_orders",
+ List.of(second.awsRecord()));
+ assertEquals(Map.of(SHARD_ID, "s2"), integration.getShardSequenceNumbers());
+ }
+
+ @Test
+ void aPollSubmittedBeforeAResetIsSkippedEvenWhenItRunsAfterTheReset() {
+ FakeStreamReader streams = new FakeStreamReader(CheckpointLifetime.PROCESS);
+ RedshiftService redshiftService = mock(RedshiftService.class);
+ RedshiftZeroEtlWriter writer = mock(RedshiftZeroEtlWriter.class);
+ Integration integration = streamingIntegration();
+ DynamoDbStreamReader.Record first = streamRecord("s1");
+ streams.shard(SHARD_ID, null).add(first);
+ RedshiftDynamoDbZeroEtlConsumer consumer =
+ new RedshiftDynamoDbZeroEtlConsumer(streams, dynamoDb, redshiftService, writer);
+ Runnable submittedBeforeReset = consumer.pollTask(integration);
+
+ consumer.beforeReset();
+ consumer.clear();
+ consumer.afterReset();
+ submittedBeforeReset.run();
+
+ verifyNoInteractions(writer);
+ verify(redshiftService, never()).updateIntegrationRuntime(any(), any(), any(), anyBoolean(), any());
+
+ consumer.pollTask(integration).run();
+
+ verify(writer).writeBatch(integration.getAccountId(), "warehouse", "floci_zetl_orders",
+ List.of(first.awsRecord()));
+ verify(redshiftService).updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),
+ Map.of(SHARD_ID, "s1"), true, null);
+ }
+
+ @Test
+ void shutdownWaitsForThePollInProgressSoItsCheckpointIsSavedFirst() throws Exception {
+ FakeStreamReader streams = new FakeStreamReader(CheckpointLifetime.PROCESS);
+ RedshiftService redshiftService = mock(RedshiftService.class);
+ RedshiftZeroEtlWriter writer = mock(RedshiftZeroEtlWriter.class);
+ List<String> events = new CopyOnWriteArrayList<>();
+ CountDownLatch writing = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ writing.countDown();
+ release.await(10, TimeUnit.SECONDS);
+ return null;
+ }).when(writer).writeBatch(any(), any(), any(), any());
+ doAnswer(invocation -> events.add("checkpoint saved"))
+ .when(redshiftService).updateIntegrationRuntime(any(), any(), any(), anyBoolean(), any());
+ Vertx vertx = mock(Vertx.class);
+ when(vertx.setPeriodic(anyLong(), any())).thenReturn(7L);
+ Integration integration = streamingIntegration();
+ streams.shard(SHARD_ID, null).add(streamRecord("s1"));
+ RedshiftDynamoDbZeroEtlConsumer consumer = new RedshiftDynamoDbZeroEtlConsumer(vertx, streams, dynamoDb,
+ redshiftService, writer, MAPPER, mock(EmulatorConfig.class, RETURNS_DEEP_STUBS));
+ consumer.startPolling(integration);
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<Handler<Long>> tick = ArgumentCaptor.forClass(Handler.class);
+ verify(vertx).setPeriodic(anyLong(), tick.capture());
+ Thread shutdown = new Thread(() -> {
+ consumer.shutdown();
+ events.add("shutdown returned");
+ });
+
+ try {
+ tick.getValue().handle(7L);
+ assertTrue(writing.await(10, TimeUnit.SECONDS), "the tick hands the poll to the poll executor");
+ shutdown.start();
+ assertEquals(Thread.State.TIMED_WAITING, awaitTimedWaitingOrTerminated(shutdown),
+ "shutdown() returned while a poll was still writing");
+ release.countDown();
+ shutdown.join(10_000);
+ } finally {
+ release.countDown();
+ }
+
+ assertEquals(List.of("checkpoint saved", "shutdown returned"), events);
+ verify(redshiftService).updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),
+ Map.of(SHARD_ID, "s1"), true, null);
+ }
+
+ @Test
+ void aTickFromATimerCanceledByAResetDoesNotPoll() {
+ FakeStreamReader streams = new FakeStreamReader(CheckpointLifetime.PROCESS);
+ RedshiftService redshiftService = mock(RedshiftService.class);
+ RedshiftZeroEtlWriter writer = mock(RedshiftZeroEtlWriter.class);
+ Vertx vertx = mock(Vertx.class);
+ when(vertx.setPeriodic(anyLong(), any())).thenReturn(7L);
+ Integration integration = streamingIntegration();
+ streams.shard(SHARD_ID, null).add(streamRecord("s1"));
+ RedshiftDynamoDbZeroEtlConsumer consumer = new RedshiftDynamoDbZeroEtlConsumer(vertx, streams, dynamoDb,
+ redshiftService, writer, MAPPER, mock(EmulatorConfig.class, RETURNS_DEEP_STUBS));
+ consumer.startPolling(integration);
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<Handler<Long>> tick = ArgumentCaptor.forClass(Handler.class);
+ verify(vertx).setPeriodic(anyLong(), tick.capture());
+
+ consumer.beforeReset();
+ consumer.clear();
+ consumer.afterReset();
+ tick.getValue().handle(7L);
+ // shutdown() waits for every submitted poll to end, so anything the tick submitted has run.
+ consumer.shutdown();
+
+ assertTrue(streams.iteratorRequests.isEmpty(), "the stale tick read the stream");
+ verifyNoInteractions(writer);
+ verify(redshiftService, never()).updateIntegrationRuntime(any(), any(), any(), anyBoolean(), any());
+ }
+
+ /** Observes the thread park in a timed wait, or end, instead of sleeping to hope for either. */
+ private static Thread.State awaitTimedWaitingOrTerminated(Thread thread) {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ Thread.State state = thread.getState();
+ while (state != Thread.State.TIMED_WAITING && state != Thread.State.TERMINATED) {
+ if (System.nanoTime() > deadline) {
+ fail("the thread neither parked in a timed wait nor ended");
+ }
+ Thread.onSpinWait();
+ state = thread.getState();
+ }
+ return state;
+ }
+
+ /** Observes the reset thread park on the poll's lock instead of sleeping to hope for it. */
+ private static void awaitWaiting(Thread thread) {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (thread.getState() != Thread.State.WAITING) {
+ if (System.nanoTime() > deadline) {
+ fail("beforeReset() returned or never waited while a write was in progress");
+ }
+ Thread.onSpinWait();
+ }
+ }
+
private static Integration streamingIntegration() {
Integration integration = integration("orders");
integration.setBackfillCompleted(true);