Light, fluffy, and always free - The AWS Local Emulator alternative
refactor(dynamodb): scope runtime work to the selected backend (#4401)
Mario Toffiad658c75a413d
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.mdindex 0aa5ed41a9..cbf44c7670 100644--- a/docs/services/dynamodb.md+++ b/docs/services/dynamodb.md@@ -59,6 +59,17 @@ Redshift zero-ETL integrations can consume these stream records directly. See th[Redshift DynamoDB zero-ETL](redshift.md#dynamodb-zero-etl) section for the supported target,landing table, checkpoint, and retry behavior.+Stream records are held in memory. `POST /_floci/state/reset` removes every stream along with the+tables, so `ListStreams` and `DescribeStream` no longer return the streams of tables created before+the reset.++## Time to Live++With TTL enabled, a sweep runs every 60 seconds and deletes the items whose TTL attribute holds an+epoch time in the past. Each deletion writes a `REMOVE` record to the table's stream and is+forwarded to an active Kinesis streaming destination. Between sweeps, reads already leave expired+items out.+## Configuration| Variable | Default | Description |diff --git a/src/main/java/io/github/hectorvent/floci/core/common/Resettable.java b/src/main/java/io/github/hectorvent/floci/core/common/Resettable.javaindex 7a23bd59fb..90960d33fe 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/Resettable.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/Resettable.java@@ -7,6 +7,12 @@ package io.github.hectorvent.floci.core.common;* controller never holds StorageFactory's monitor while invoking them.*/public interface Resettable {+ /**+ * Throw to refuse an emulator-wide reset. Runs on every service before anything is stopped or+ * cleared, so a refusal leaves all state intact.+ */+ default void checkReset() {}+/** Stop accepting work and drain active writes before any storage is cleared. */default void beforeReset() {}diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoController.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoController.javaindex b8b4d63757..8f90b27a1d 100644--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoController.java+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoController.java@@ -156,17 +156,21 @@ public class EmulatorInfoController {}private synchronized void performReset() {- // Containers first: they are tracked independently of StorageBackend, so this can run- // in any order relative to the storage wipe below, but stopping them here means a- // client's reset actually reflects a clean slate instead of leaving Batch, CodeBuild,- // or SageMaker containers running with no record of them left in the store.- ContainerTeardowns.stopAll(containerTeardowns, LOG);// Resolve live instances before taking any storage locks. Serialize resets so one reset// cannot resume a publisher while another is still wiping its storage.List<Resettable> services = new ArrayList<>();for (Resettable service : resettables) {services.add(service);}+ // Every service may refuse before the first destructive step, so a refusal changes nothing.+ for (Resettable service : services) {+ service.checkReset();+ }+ // Containers next: they are tracked independently of StorageBackend, so this can run+ // in any order relative to the storage wipe below, but stopping them here means a+ // client's reset actually reflects a clean slate instead of leaving Batch, CodeBuild,+ // or SageMaker containers running with no record of them left in the store.+ ContainerTeardowns.stopAll(containerTeardowns, LOG);RuntimeException failure = null;try {for (Resettable service : services) {diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.javaindex 91d1b46fe4..1d4b60363c 100644--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java@@ -21,6 +21,7 @@ import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContimport io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;import io.github.hectorvent.floci.services.elasticache.proxy.ElastiCacheProxyManager;import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbRuntime;import io.github.hectorvent.floci.services.lambda.DynamoDbStreamsEventSourcePoller;import io.github.hectorvent.floci.services.lambda.KinesisEventSourcePoller;import io.github.hectorvent.floci.services.lambda.SqsEventSourcePoller;@@ -36,6 +37,7 @@ import io.github.hectorvent.floci.services.memorydb.container.MemoryDbContainerMimport io.github.hectorvent.floci.services.memorydb.proxy.MemoryDbProxyManager;import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;+import io.github.hectorvent.floci.services.redshift.RedshiftDynamoDbZeroEtlConsumer;import io.github.hectorvent.floci.services.timestreaminfluxdb.TimestreamInfluxDbService;import io.quarkus.runtime.Quarkus;import io.quarkus.runtime.ShutdownDelayInitiatedEvent;@@ -106,6 +108,8 @@ public class EmulatorLifecycle {private final StepFunctionsService stepFunctionsService;private final Instance<ContainerTeardown> containerTeardowns;private final PersistentPathValidator persistentPathValidator;+ private final DynamoDbRuntime dynamoDbRuntime;+ private final RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer;@Injectpublic EmulatorLifecycle(StorageFactory storageFactory, ServiceRegistry serviceRegistry,@@ -142,7 +146,9 @@ public class EmulatorLifecycle {SchemaCreationWorker schemaCreationWorker,StepFunctionsService stepFunctionsService,Instance<ContainerTeardown> containerTeardowns,- PersistentPathValidator persistentPathValidator) {+ PersistentPathValidator persistentPathValidator,+ DynamoDbRuntime dynamoDbRuntime,+ RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer) {this.storageFactory = storageFactory;this.serviceRegistry = serviceRegistry;this.config = config;@@ -179,6 +185,8 @@ public class EmulatorLifecycle {this.stepFunctionsService = stepFunctionsService;this.containerTeardowns = containerTeardowns;this.persistentPathValidator = persistentPathValidator;+ this.dynamoDbRuntime = dynamoDbRuntime;+ this.redshiftZeroEtlConsumer = redshiftZeroEtlConsumer;}void onStart(@Observes StartupEvent ignored) {@@ -224,10 +232,13 @@ public class EmulatorLifecycle {schemaCreationWorker.rehydrateSchemas();stepFunctionsService.abortAbandonedExecutions();+ // The selected DynamoDB backend is ready before any persisted stream consumer reads it.+ dynamoDbRuntime.start();sqsPoller.startPersistedPollers();kinesisPoller.startPersistedPollers();dynamodbStreamsPoller.startPersistedPollers();pipesService.startPersistedPollers();+ redshiftZeroEtlConsumer.startPersistedIntegrations();rdsService.restorePersistedRuntime();if (config.services().timestreamInfluxdb().enabled()) {timestreamInfluxDbService.restorePersistedRuntime();@@ -343,6 +354,9 @@ public class EmulatorLifecycle {// SIGTERM grace window and trigger SIGKILL; if the flush ran last it would be skipped and// in-memory (hybrid) data would be lost on an otherwise-graceful shutdown. shutdownAll()// still runs at the end to stop the flush schedulers and capture any shutdown-time writes.+ // DynamoDB stream consumers stop first, so the flush holds their final checkpoints.+ runCleanup("DynamoDB Streams poller", dynamodbStreamsPoller::shutdown);+ runCleanup("Redshift zero-ETL consumer", redshiftZeroEtlConsumer::shutdown);runCleanup("storage flush", storageFactory::flushAll);runCleanup("EC2 metadata server", () -> {if (isMetadataServerNeeded()) {@@ -372,6 +386,7 @@ public class EmulatorLifecycle {// EC2 instances, in-flight build/job containers). Runs before shutdownAll() so any// state written while stopping is captured by the final flush.ContainerTeardowns.stopAll(containerTeardowns, LOG);+ runCleanup("DynamoDB backend", dynamoDbRuntime::stop);runCleanup("storage shutdown", storageFactory::shutdownAll);LOG.info("=== AWS Local Emulator Stopped ===");diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntime.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntime.javanew file mode 100644index 0000000000..44021473c0--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntime.java@@ -0,0 +1,68 @@+package io.github.hectorvent.floci.services.dynamodb;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.Resettable;+import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbBackendLifecycle;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;++/**+ * Drives the selected DynamoDB backend through the emulator lifecycle: readiness at startup,+ * reset preflight, quiescing and cleanup, and shutdown. Does nothing while DynamoDB is disabled.+ */+@ApplicationScoped+public class DynamoDbRuntime implements Resettable {++ private final DynamoDbBackendLifecycle backend;+ private final EmulatorConfig config;++ @Inject+ public DynamoDbRuntime(DynamoDbBackendLifecycle backend, EmulatorConfig config) {+ this.backend = backend;+ this.config = config;+ }++ public void start() {+ if (enabled()) {+ backend.start();+ }+ }++ public void stop() {+ if (enabled()) {+ backend.stop();+ }+ }++ @Override+ public void checkReset() {+ if (enabled()) {+ backend.checkReset();+ }+ }++ @Override+ public void beforeReset() {+ if (enabled()) {+ backend.beforeReset();+ }+ }++ @Override+ public void clear() {+ if (enabled()) {+ backend.reset();+ }+ }++ @Override+ public void afterReset() {+ if (enabled()) {+ backend.afterReset();+ }+ }++ private boolean enabled() {+ return config.services().dynamodb().enabled();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 14ed108f3f..61b9951c30 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -227,6 +227,26 @@ public class DynamoDbService {recoverInterruptedJobs();}+ /** Emulator reset: drops the in-process item cache, locks and idempotency tokens, and stream state. */+ public void clearProcessState() {+ itemsByTable.clear();+ itemLocks.clear();+ txIdempotency.clear();+ if (streamService != null) {+ streamService.clear();+ }+ if (kinesisForwarder != null) {+ kinesisForwarder.clear();+ }+ }++ /** Shutdown: ends Kinesis forwarding drains so none writes to Kinesis during the final storage flush. */+ public void stopKinesisForwarding() {+ if (kinesisForwarder != null) {+ kinesisForwarder.clear();+ }+ }+private void loadPersistedItems() {if (itemStore == null) return;Map<String, TableDefinition> persistedTables = persistedTablesByScopedKey();@@ -2687,7 +2707,8 @@ public class DynamoDbService {}record ExpiredTableScan(String rawKey, String accountId, String storageKey, String region,- TableDefinition table, List<String> itemKeys) {}+ TableDefinition table, List<String> itemKeys,+ ConcurrentSkipListMap<String, JsonNode> items) {}void deleteExpiredItems() {deleteScannedItems(scanExpiredItems());@@ -2726,7 +2747,7 @@ public class DynamoDbService {String accountId = slash >= 0 ? rawKey.substring(0, slash) : null;String storageKey = slash >= 0 ? rawKey.substring(slash + 1) : rawKey;String region = storageKey.split("::", 2)[0];- scans.add(new ExpiredTableScan(rawKey, accountId, storageKey, region, table, expiredKeys));+ scans.add(new ExpiredTableScan(rawKey, accountId, storageKey, region, table, expiredKeys, items));}return scans;}@@ -2734,13 +2755,14 @@ public class DynamoDbService {void deleteScannedItems(List<ExpiredTableScan> scans) {int totalDeleted = 0;for (ExpiredTableScan scan : scans) {- ConcurrentSkipListMap<String, JsonNode> items = itemsByTable.get(scan.rawKey());- if (items == null) {- continue;- }+ ConcurrentSkipListMap<String, JsonNode> items = scan.items();int deletedForTable = 0;for (String itemKey : scan.itemKeys()) {+ // A table deleted, or deleted and recreated, since the scan or during this sweep: its items are not this scan's.+ if (itemsByTable.get(scan.rawKey()) != items) {+ break;+ }JsonNode removed = withScopedItemLock(scan.rawKey(), itemKey, () -> {JsonNode current = items.get(itemKey);if (current == null || !isExpired(current, scan.table())) {@@ -2762,7 +2784,15 @@ public class DynamoDbService {}}if (deletedForTable > 0) {- persistItemsForAccount(scan.accountId(), scan.storageKey(), items);+ // A DeleteTable during the loop detached this map; writing it back would revive its items.+ // Persisting under the entry's lock holds off DeleteTable, which removes the entry before+ // it deletes the stored items.+ itemsByTable.computeIfPresent(scan.rawKey(), (key, live) -> {+ if (live == items) {+ persistItemsForAccount(scan.accountId(), scan.storageKey(), items);+ }+ return live;+ });totalDeleted += deletedForTable;}}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.javaindex f5327c2311..95bbb67d58 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java@@ -146,6 +146,13 @@ public class DynamoDbStreamService {}}+ /** Emulator reset: drops every stream and its records. Sequence numbers stay monotonic. */+ public void clear() {+ streams.clear();+ records.clear();+ streamRecordCounts.clear();+ }+public void captureEvent(String eventName, JsonNode oldItem, JsonNode newItem,TableDefinition table, String region) {StreamDescription sd = streams.get(table.getTableArn());diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlService.javaindex 018c2c8348..b9676a3ef0 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlService.java@@ -1,6 +1,5 @@package io.github.hectorvent.floci.services.dynamodb;-import jakarta.annotation.PostConstruct;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -17,6 +16,7 @@ public class DynamoDbTtlService {private final DynamoDbService dynamoDbService;private final ScheduledExecutorService scheduler;+ private boolean paused;@Injectpublic DynamoDbTtlService(DynamoDbService dynamoDbService) {@@ -28,14 +28,35 @@ public class DynamoDbTtlService {});}- @PostConstruct- void init() {- scheduler.scheduleAtFixedRate(dynamoDbService::deleteExpiredItems, 60, 60, TimeUnit.SECONDS);+ public void start() {+ scheduler.scheduleAtFixedRate(this::sweep, 60, 60, TimeUnit.SECONDS);LOG.infov("DynamoDB TTL sweeper scheduled (60s interval)");}+ synchronized void sweep() {+ if (paused) {+ return;+ }+ try {+ dynamoDbService.deleteExpiredItems();+ } catch (RuntimeException e) {+ LOG.warnv(e, "DynamoDB TTL sweep failed; retrying on the next run");+ }+ }++ /** Skips sweeps until {@link #resume()}, first waiting for a sweep already running to finish. */+ public synchronized void pause() {+ paused = true;+ }++ public synchronized void resume() {+ paused = false;+ }++ /** Stops sweeping for good, first waiting for a sweep already running to finish. */@PreDestroy- void shutdown() {+ public void stop() {+ pause();scheduler.shutdownNow();}}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.javaindex d5a71c2d4b..02aaf82f7f 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java@@ -374,6 +374,12 @@ public class KinesisStreamingForwarder {}}+ /** Emulator reset and shutdown: discard every destination's buffer, stop its drains, and drop its state. */+ public void clear() {+ states.values().forEach(this::invalidate);+ states.clear();+ }+@PreDestroypublic void shutdown() {scheduler.shutdown();diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendLifecycle.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendLifecycle.javanew file mode 100644index 0000000000..f7c5e33e3c--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendLifecycle.java@@ -0,0 +1,23 @@+package io.github.hectorvent.floci.services.dynamodb.backend;++/** The selected engine's process lifecycle, driven by {@code DynamoDbRuntime}. */+public interface DynamoDbBackendLifecycle {++ /** Initializes the engine and its maintenance work; throws when it cannot serve requests. */+ void start();++ /** Throws to refuse an emulator-wide reset; runs before anything is stopped or cleared. */+ void checkReset();++ /** Quiesces maintenance work before Floci storage is cleared. */+ void beforeReset();++ /** Drops the engine's process state; runs after Floci storage was cleared. */+ void reset();++ /** Resumes maintenance work at the end of a reset; may run without a preceding {@link #beforeReset()}. */+ void afterReset();++ /** Stops maintenance work and closes clients. */+ void stop();+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelector.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelector.javaindex 4c18b42c72..76a5eb04ab 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelector.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelector.java@@ -9,7 +9,7 @@ import jakarta.inject.Inject;* The one immutable DynamoDB backend selection. Native is the only engine today; a configurable* choice arrives with the alternative engine. {@link NativeDynamoDbBackend} is typed to its concrete* class and {@link DynamoDbApiStreamReader} is not a bean, so these producers are the only- * unqualified beans for the four seam interfaces.+ * unqualified beans for the five seam interfaces.*/@ApplicationScopedpublic class DynamoDbBackendSelector {@@ -36,6 +36,11 @@ public class DynamoDbBackendSelector {return nativeBackend;}+ @Produces+ DynamoDbBackendLifecycle lifecycle() {+ return nativeBackend;+ }+/** Native stream history lives only in memory, so committed progress ends with the process. */@ProducesDynamoDbStreamReader streamReader() {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/NativeDynamoDbBackend.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/NativeDynamoDbBackend.javaindex 230db5390a..71cd5998d5 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/NativeDynamoDbBackend.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/NativeDynamoDbBackend.java@@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlService;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbJsonHandler;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbStreamsJsonHandler;import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;@@ -12,6 +14,7 @@ import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;+import io.quarkus.arc.ClientProxy;import jakarta.enterprise.context.ApplicationScoped;import jakarta.enterprise.inject.Typed;import jakarta.inject.Inject;@@ -27,23 +30,66 @@ import java.util.Optional;*/@ApplicationScoped@Typed(NativeDynamoDbBackend.class)-public class NativeDynamoDbBackend implements DynamoDbOperations, DynamoDbItemAccess, DynamoDbTableAccess {+public class NativeDynamoDbBackend implements DynamoDbOperations, DynamoDbItemAccess, DynamoDbTableAccess,+ DynamoDbBackendLifecycle {private final NativeDynamoDbJsonHandler jsonHandler;private final NativeDynamoDbStreamsJsonHandler streamsHandler;private final DynamoDbService dynamoDbService;+ private final DynamoDbStreamService streamService;+ private final DynamoDbTtlService ttlService;private final ObjectMapper objectMapper;@Injectpublic NativeDynamoDbBackend(NativeDynamoDbJsonHandler jsonHandler,NativeDynamoDbStreamsJsonHandler streamsHandler,- DynamoDbService dynamoDbService, ObjectMapper objectMapper) {+ DynamoDbService dynamoDbService, DynamoDbStreamService streamService,+ DynamoDbTtlService ttlService, ObjectMapper objectMapper) {this.jsonHandler = jsonHandler;this.streamsHandler = streamsHandler;this.dynamoDbService = dynamoDbService;+ this.streamService = streamService;+ this.ttlService = ttlService;this.objectMapper = objectMapper;}+ /**+ * Resolves the engine beans so their constructors load persisted items, recover interrupted+ * jobs and rebuild stream buffers now, before any consumer reads, then schedules the TTL sweep.+ */+ @Override+ public void start() {+ ClientProxy.unwrap(dynamoDbService);+ ClientProxy.unwrap(streamService);+ ttlService.start();+ }++ /** A native reset is always allowed. */+ @Override+ public void checkReset() {+ }++ @Override+ public void beforeReset() {+ ttlService.pause();+ }++ @Override+ public void reset() {+ dynamoDbService.clearProcessState();+ }++ @Override+ public void afterReset() {+ ttlService.resume();+ }++ @Override+ public void stop() {+ ttlService.stop();+ dynamoDbService.stopKinesisForwarding();+ }+@Overridepublic Reply execute(Call call) throws Exception {String region = call.scope().region();diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.javaindex a9b311df9d..a22c52dc9e 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java@@ -75,7 +75,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {final Set<String> stopped = ConcurrentHashMap.newKeySet();/** {@code uuid:shardId} of shards a mapping read to their end; re-derived from committed progress. */private final Set<String> finishedShards = ConcurrentHashMap.newKeySet();- private volatile boolean resetting;+ /** Set during a reset and after shutdown: no invocation starts and no checkpoint is saved. */+ private volatile boolean quiesced;private final ExecutorService pollExecutor = Executors.newCachedThreadPool(r -> {Thread t = new Thread(r, "dynamodb-streams-esm-poller");t.setDaemon(true);@@ -185,8 +186,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {}}+ /**+ * Stops polling for good. Taking the monitor drains an in-flight advanceCheckpoint save, and no+ * checkpoint is saved after this returns. Idempotent, so it may run again from {@code @PreDestroy}.+ */@PreDestroy- void shutdown() {+ public synchronized void shutdown() {+ quiesced = true;pollExecutor.shutdownNow();timerIds.values().forEach(vertx::cancelTimer);timerIds.clear();@@ -195,12 +201,12 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {@Overridepublic synchronized void beforeReset() {// Taking the monitor drains an in-flight advanceCheckpoint save; it is released before the wipe.- resetting = true;+ quiesced = true;}@Overridepublic synchronized void afterReset() {- resetting = false;+ quiesced = pollExecutor.isShutdown();}public void clear() {@@ -326,7 +332,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {boolean enabled = esmStore.getForAccount(esm.getAccountId(), esm.getUuid()).map(EventSourceMapping::isEnabled).orElse(false);- if (resetting || !enabled) {+ if (quiesced || !enabled) {return;}@@ -755,7 +761,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {}private boolean isStopped(EventSourceMapping esm) {- return resetting || stopped.contains(esm.getUuid()) || !exists(esm);+ return quiesced || stopped.contains(esm.getUuid()) || !exists(esm);}private boolean exists(EventSourceMapping esm) {diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.javaindex dd214e8f97..6c4b61b6a0 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.services.dynamodb.DynamoDbFacade;import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbItemAccess.ScanPage;import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbOperations.Scope;@@ -16,11 +17,9 @@ import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;import io.github.hectorvent.floci.services.redshift.model.Integration;import io.github.hectorvent.floci.services.redshiftdata.RedshiftZeroEtlWriter;-import io.quarkus.runtime.StartupEvent;import io.vertx.core.Vertx;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;-import jakarta.enterprise.event.Observes;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -33,9 +32,11 @@ import java.util.Set;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.locks.ReentrantReadWriteLock;@ApplicationScoped-public class RedshiftDynamoDbZeroEtlConsumer {+public class RedshiftDynamoDbZeroEtlConsumer implements Resettable {private static final Logger LOG = Logger.getLogger(RedshiftDynamoDbZeroEtlConsumer.class);private static final int BATCH_SIZE = 100;@@ -56,6 +57,11 @@ public class RedshiftDynamoDbZeroEtlConsumer {thread.setDaemon(true);return thread;});+ /** Polls hold the read lock; a reset takes the write lock to wait them out. */+ private final ReentrantReadWriteLock quiesceLock = new ReentrantReadWriteLock();+ private volatile boolean quiesced;+ /** Completed resets: a poll writes only while none is in progress and none completed since its submission. */+ private volatile long resets;@Injectpublic RedshiftDynamoDbZeroEtlConsumer(Vertx vertx,@@ -92,10 +98,6 @@ public class RedshiftDynamoDbZeroEtlConsumer {this.pollIntervalMs = pollIntervalMs;}- void onStart(@Observes StartupEvent event) {- startPersistedIntegrations();- }-/*** Starts the persisted integrations, first discarding their stream progress when it lasts only* for the process: native stream history is volatile, so a sequence number saved by a previous@@ -116,8 +118,8 @@ public class RedshiftDynamoDbZeroEtlConsumer {if (vertx == null || timerIds.containsKey(integration.getIntegrationArn())) {return;}- long timerId = vertx.setPeriodic(pollIntervalMs, ignored -> pollAnd(integration));- timerIds.put(integration.getIntegrationArn(), timerId);+ timerIds.put(integration.getIntegrationArn(),+ vertx.setPeriodic(pollIntervalMs, timerId -> pollAnd(integration, timerId)));}public void stopPolling(String integrationArn) {@@ -130,6 +132,10 @@ public class RedshiftDynamoDbZeroEtlConsumer {}void pollOnce(Integration integration) {+ unlessQuiesced(resets, () -> poll(integration));+ }++ private void poll(Integration integration) {writer.createLandingTable(integration.getAccountId(), integration.getTargetClusterIdentifier(),integration.getLandingTableName());if (!integration.isBackfillCompleted()) {@@ -261,7 +267,22 @@ public class RedshiftDynamoDbZeroEtlConsumer {}}- public void reset() {+ /**+ * Waits for the poll in progress, then keeps every poll from writing until {@link #afterReset()},+ * and a poll submitted before that from writing at all.+ */+ @Override+ public void beforeReset() {+ quiesceLock.writeLock().lock();+ try {+ quiesced = true;+ } finally {+ quiesceLock.writeLock().unlock();+ }+ }++ @Override+ public void clear() {for (String integrationArn : timerIds.keySet()) {stopPolling(integrationArn);}@@ -269,34 +290,86 @@ public class RedshiftDynamoDbZeroEtlConsumer {finishedShards.clear();}+ /** Lets the polls submitted from now on write again. */+ @Override+ public void afterReset() {+ quiesceLock.writeLock().lock();+ try {+ resets++;+ quiesced = false;+ } finally {+ quiesceLock.writeLock().unlock();+ }+ }++ /**+ * Stops every integration for good, waiting up to five seconds for a poll in progress so the storage+ * flush that follows holds its checkpoint. Idempotent, so it may run again from {@code @PreDestroy}.+ */@PreDestroy- void onStop() {- reset();- pollExecutor.shutdownNow();+ public void shutdown() {+ clear();+ pollExecutor.shutdown();+ try {+ if (!pollExecutor.awaitTermination(5, TimeUnit.SECONDS)) {+ pollExecutor.shutdownNow();+ LOG.warnv("A zero-ETL poll was still running at shutdown, so its last checkpoint may not be flushed");+ }+ } catch (InterruptedException e) {+ pollExecutor.shutdownNow();+ Thread.currentThread().interrupt();+ }+ }++ /** Runs the section unless a reset is in progress or one completed since {@code submittedAt} was read. */+ private void unlessQuiesced(long submittedAt, Runnable section) {+ quiesceLock.readLock().lock();+ try {+ if (!quiesced && resets == submittedAt) {+ section.run();+ }+ } finally {+ quiesceLock.readLock().unlock();+ }}- private void pollAnd(Integration integration) {+ /**+ * Submits the tick's poll only while its timer is still registered, building the task first: {@code clear()}+ * removes the timer before {@code afterReset()} counts the reset, both on the reset thread, so a task that+ * read the new count sees its timer gone and one that read the old count is rejected by {@code unlessQuiesced}.+ */+ private void pollAnd(Integration integration, long timerId) {+ Runnable task = pollTask(integration);String integrationArn = integration.getIntegrationArn();+ if (!Long.valueOf(timerId).equals(timerIds.get(integrationArn))) {+ return;+ }if (activePolls.putIfAbsent(integrationArn, Boolean.TRUE) != null) {return;}- pollExecutor.submit(() -> {+ pollExecutor.submit(task);+ }++ /** The poll {@code pollAnd} submits, bound to the resets completed at its submission. */+ Runnable pollTask(Integration integration) {+ long submittedAt = resets;+ return () -> {try {- pollSafely(integration);+ pollSafely(integration, submittedAt);} finally {- activePolls.remove(integrationArn);+ activePolls.remove(integration.getIntegrationArn());}- });+ };}- private void pollSafely(Integration integration) {+ private void pollSafely(Integration integration, long submittedAt) {try {- pollOnce(integration);+ unlessQuiesced(submittedAt, () -> poll(integration));} catch (Exception e) {LOG.warnv(e, "Zero-ETL polling failed for integration {0}", integration.getIntegrationArn());try {- redshiftService.updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),- integration.getShardSequenceNumbers(), false, e.getMessage());+ unlessQuiesced(submittedAt, () -> redshiftService.updateIntegrationRuntime(integration.getAccountId(),+ integration.getIntegrationArn(), integration.getShardSequenceNumbers(), false, e.getMessage()));} catch (Exception updateError) {LOG.warnv(updateError, "Could not persist zero-ETL failure for integration {0}",integration.getIntegrationArn());diff --git a/src/test/java/io/github/hectorvent/floci/core/common/IamConditionContextResolverTest.java b/src/test/java/io/github/hectorvent/floci/core/common/IamConditionContextResolverTest.javaindex 4d544422f9..59b55eabeb 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/IamConditionContextResolverTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/IamConditionContextResolverTest.java@@ -56,7 +56,7 @@ class IamConditionContextResolverTest {@BeforeEachvoid setUp() {dynamoDbService = mock(DynamoDbService.class);- NativeDynamoDbBackend dynamoDbBackend = new NativeDynamoDbBackend(null, null, dynamoDbService, mapper);+ NativeDynamoDbBackend dynamoDbBackend = new NativeDynamoDbBackend(null, null, dynamoDbService, null, null, mapper);dynamoDbFacadeInstance = mock(Instance.class);when(dynamoDbFacadeInstance.isResolvable()).thenReturn(true);when(dynamoDbFacadeInstance.get()).thenReturn(new DynamoDbFacade(dynamoDbBackend, dynamoDbBackend,diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerIntegrationTest.javaindex 4e850d0b7b..53ecfa5af5 100644--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerIntegrationTest.java@@ -1,15 +1,17 @@package io.github.hectorvent.floci.lifecycle;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;-import com.fasterxml.jackson.databind.ObjectMapper;-import com.fasterxml.jackson.databind.JsonNode;+import io.restassured.response.ValidatableResponse;import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.ValueSource;-import org.junit.jupiter.api.Test;import java.util.List;+import java.util.UUID;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.*;@@ -152,6 +154,35 @@ class EmulatorInfoControllerIntegrationTest {.body("schemas[0]", equalTo("urn:ietf:params:scim:schemas:core:2.0:ServiceProviderConfig"));}+ @Test+ void stateResetClearsDynamoDbStreams() {+ String tableName = "reset-streams-" + UUID.randomUUID();+ dynamoDb("DynamoDB_20120810.CreateTable", """+ {"TableName": "%s",+ "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}],+ "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"}],+ "BillingMode": "PAY_PER_REQUEST",+ "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}}+ """.formatted(tableName))+ .statusCode(200);+ String listStreams = "{\"TableName\": \"" + tableName + "\"}";+ dynamoDb("DynamoDBStreams_20120810.ListStreams", listStreams).body("Streams", hasSize(1));++ given().when().post("/_floci/state/reset").then().statusCode(200);++ dynamoDb("DynamoDBStreams_20120810.ListStreams", listStreams).statusCode(200).body("Streams", hasSize(0));+ }++ private static ValidatableResponse dynamoDb(String target, String body) {+ return given()+ .header("X-Amz-Target", target)+ .contentType("application/x-amz-json-1.0")+ .body(body)+ .when()+ .post("/")+ .then();+ }+@Testvoid stateReset_clearsDatabaseState() {// 1. Put SSM parameterdiff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerTest.javaindex 1023f6964f..5fdcba6e69 100644--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerTest.java+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorInfoControllerTest.java@@ -18,12 +18,16 @@ import org.mockito.junit.jupiter.MockitoExtension;import java.util.List;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertSame;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.inOrder;+import static org.mockito.Mockito.lenient;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;+import static org.mockito.Mockito.verifyNoMoreInteractions;import static org.mockito.Mockito.when;@ExtendWith(MockitoExtension.class)@@ -148,4 +152,21 @@ class EmulatorInfoControllerTest {verify(resettable, never()).clear();verify(later, never()).clear();}++ @Test+ void resetRefusedByPreflightChangesNothing() {+ Resettable refusing = mock(Resettable.class);+ IllegalStateException refusal = new IllegalStateException("backend refuses reset");+ doThrow(refusal).when(refusing).checkReset();+ lenient().when(containerTeardowns.iterator()).thenReturn(List.of(sageMakerTeardown).iterator());+ when(resettables.iterator()).thenReturn(List.of(resettable, refusing).iterator());++ assertSame(refusal, assertThrows(IllegalStateException.class, controller::reset));++ verify(resettable).checkReset();+ verify(refusing).checkReset();+ verifyNoMoreInteractions(resettable, refusing);+ verify(sageMakerTeardown, never()).stopManagedContainers();+ verifyNoInteractions(storageFactory);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.javaindex bdb65112ff..61b0c6d738 100644--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;import io.github.hectorvent.floci.lifecycle.inithook.InitializationHooksRunner;import io.github.hectorvent.floci.services.appsync.graphql.SchemaCreationWorker;import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbRuntime;import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;import io.github.hectorvent.floci.services.ecs.container.EcsTaskRoleCredentialsServer;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;@@ -28,6 +29,7 @@ import io.github.hectorvent.floci.services.pipes.PipesService;import io.github.hectorvent.floci.services.rds.RdsService;import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;+import io.github.hectorvent.floci.services.redshift.RedshiftDynamoDbZeroEtlConsumer;import io.github.hectorvent.floci.services.stepfunctions.StepFunctionsService;import io.github.hectorvent.floci.services.timestreaminfluxdb.TimestreamInfluxDbService;import io.quarkus.runtime.Quarkus;@@ -113,6 +115,8 @@ class EmulatorLifecycleTest {@Mock private SchemaCreationWorker schemaCreationWorker;@Mock private StepFunctionsService stepFunctionsService;@Mock private Instance<ContainerTeardown> containerTeardowns;+ @Mock private DynamoDbRuntime dynamoDbRuntime;+ @Mock private RedshiftDynamoDbZeroEtlConsumer redshiftZeroEtlConsumer;private EmulatorLifecycle emulatorLifecycle;@@ -151,7 +155,8 @@ class EmulatorLifecycleTest {elbV2Service, elbClassicService,initializationHooksRunner, sqsPoller, kinesisPoller, dynamodbStreamsPoller,pipesService, ec2MetadataServer, ecsTaskRoleCredentialsServer, ecrRegistryManager, flociUiManager, initLifecycleState,- schemaCreationWorker, stepFunctionsService, containerTeardowns, persistentPathValidator);+ schemaCreationWorker, stepFunctionsService, containerTeardowns, persistentPathValidator,+ dynamoDbRuntime, redshiftZeroEtlConsumer);Mockito.lenient().when(containerTeardowns.iterator()).thenReturn(java.util.Collections.emptyIterator());}@@ -181,6 +186,46 @@ class EmulatorLifecycleTest {inOrder.verify(rdsService).restorePersistedRuntime();}+ @Test+ void startsTheDynamoDbBackendAfterStorageLoadAndBeforeItsStreamConsumers() {+ stubStorageConfig();+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);++ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));++ InOrder inOrder = Mockito.inOrder(storageFactory, dynamoDbRuntime, dynamodbStreamsPoller, pipesService,+ redshiftZeroEtlConsumer);+ inOrder.verify(storageFactory).loadAll();+ inOrder.verify(dynamoDbRuntime).start();+ inOrder.verify(dynamodbStreamsPoller).startPersistedPollers();+ inOrder.verify(pipesService).startPersistedPollers();+ inOrder.verify(redshiftZeroEtlConsumer).startPersistedIntegrations();+ }++ @Test+ void stopsDynamoDbStreamConsumersBeforeTheFlushAndTheBackendBeforeStorageShutdown() {+ emulatorLifecycle.onStop(Mockito.mock(ShutdownEvent.class));++ InOrder inOrder = Mockito.inOrder(dynamodbStreamsPoller, redshiftZeroEtlConsumer, storageFactory,+ flociUiManager, dynamoDbRuntime);+ inOrder.verify(dynamodbStreamsPoller).shutdown();+ inOrder.verify(redshiftZeroEtlConsumer).shutdown();+ inOrder.verify(storageFactory).flushAll();+ inOrder.verify(flociUiManager).shutdown();+ inOrder.verify(dynamoDbRuntime).stop();+ inOrder.verify(storageFactory).shutdownAll();+ }++ @Test+ void stopsTheZeroEtlConsumerWhenTheStreamsPollerShutdownFails() {+ doThrow(new IllegalStateException("poller shutdown failed")).when(dynamodbStreamsPoller).shutdown();++ emulatorLifecycle.onStop(Mockito.mock(ShutdownEvent.class));++ verify(redshiftZeroEtlConsumer).shutdown();+ }+@Test@DisplayName("Should restore ELBv2 persisted runtime after loading storage when elbv2 is enabled")void shouldRestoreElbV2PersistedRuntimeAfterStorageLoad() {diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CfnProvisionerFixture.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CfnProvisionerFixture.javaindex 9ee2e15b0c..c20f2461e0 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CfnProvisionerFixture.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CfnProvisionerFixture.java@@ -268,7 +268,7 @@ final class CfnProvisionerFixture {discovered.add(new SnsCfnProvisioner(snsService));}if (dynamoDbService != null) {- NativeDynamoDbBackend backend = new NativeDynamoDbBackend(null, null, dynamoDbService, objectMapper);+ NativeDynamoDbBackend backend = new NativeDynamoDbBackend(null, null, dynamoDbService, null, null, objectMapper);discovered.add(new DynamoDbCfnProvisioner(new DynamoDbFacade(backend, backend,new RegionResolver("us-east-1", "000000000000"))));}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/DynamoDbCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/DynamoDbCfnProvisionerTest.javaindex 1d0b88d0fa..0059ec46a7 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/DynamoDbCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/DynamoDbCfnProvisionerTest.java@@ -46,7 +46,7 @@ class DynamoDbCfnProvisionerTest {private final DynamoDbService dynamoDb = mock(DynamoDbService.class);private final ObjectMapper mapper = new ObjectMapper();- private final NativeDynamoDbBackend backend = new NativeDynamoDbBackend(null, null, dynamoDb, mapper);+ private final NativeDynamoDbBackend backend = new NativeDynamoDbBackend(null, null, dynamoDb, null, null, mapper);private final DynamoDbCfnProvisioner provisioner = new DynamoDbCfnProvisioner(new DynamoDbFacade(backend, backend, new RegionResolver("us-east-1", "000000000000")));diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntimeTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntimeTest.javanew file mode 100644index 0000000000..d0696e49b1--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbRuntimeTest.java@@ -0,0 +1,55 @@+package io.github.hectorvent.floci.services.dynamodb;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbBackendLifecycle;+import org.junit.jupiter.api.Test;+import org.mockito.InOrder;++import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.inOrder;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verifyNoInteractions;+import static org.mockito.Mockito.verifyNoMoreInteractions;+import static org.mockito.Mockito.when;++class DynamoDbRuntimeTest {++ private final DynamoDbBackendLifecycle backend = mock(DynamoDbBackendLifecycle.class);+ private final EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ private final DynamoDbRuntime runtime = new DynamoDbRuntime(backend, config);++ @Test+ void drivesTheSelectedBackendLifecycle() {+ when(config.services().dynamodb().enabled()).thenReturn(true);++ runtime.start();+ runtime.checkReset();+ runtime.beforeReset();+ runtime.clear();+ runtime.afterReset();+ runtime.stop();++ InOrder order = inOrder(backend);+ order.verify(backend).start();+ order.verify(backend).checkReset();+ order.verify(backend).beforeReset();+ order.verify(backend).reset();+ order.verify(backend).afterReset();+ order.verify(backend).stop();+ verifyNoMoreInteractions(backend);+ }++ @Test+ void leavesTheBackendAloneWhenDynamoDbIsDisabled() {+ when(config.services().dynamodb().enabled()).thenReturn(false);++ runtime.start();+ runtime.checkReset();+ runtime.beforeReset();+ runtime.clear();+ runtime.afterReset();+ runtime.stop();++ verifyNoInteractions(backend);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.javaindex 64f9bedae7..7de9fc8cfb 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java@@ -25,11 +25,17 @@ import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;import static org.junit.jupiter.api.Assertions.*;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.verifyNoInteractions;import static org.mockito.Mockito.when;@@ -201,6 +207,127 @@ class DynamoDbTtlForwardingTest {}}+ @Test+ void ttlSweepDoesNotPersistItemsOfATableDeletedMidSweep() {+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();+ DynamoDbStreamService streamService = mock(DynamoDbStreamService.class);+ DynamoDbService svc = new DynamoDbService(+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"), streamService, null);+ seedTtlTable(svc, 1);+ doAnswer(invocation -> {+ svc.deleteTable("TtlTable", REGION);+ return null;+ }).when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any());++ svc.deleteExpiredItems();++ assertTrue(itemStore.get("us-east-1::TtlTable").isEmpty(),+ "the sweep must not write items back for a table deleted while it ran");+ }++ @Test+ void aSweepDoesNotDeleteFromATableRecreatedAfterItsScan() {+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();+ DynamoDbService svc = new DynamoDbService(+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"));+ long past = Instant.now().getEpochSecond() - 3600;+ svc.createTable("TtlTable",+ List.of(new KeySchemaElement("pk", "HASH")),+ List.of(new AttributeDefinition("pk", "S")),+ 5L, 5L, REGION);+ svc.updateTimeToLive("TtlTable", "expireAt", true, REGION);+ ObjectNode expired = mapper.createObjectNode();+ expired.set("pk", stringAttr("row-0"));+ expired.set("expireAt", numberAttr(past));+ svc.putItem("TtlTable", expired, REGION);+ List<DynamoDbService.ExpiredTableScan> scans = svc.scanExpiredItems();+ assertEquals(1, scans.stream().mapToInt(scan -> scan.itemKeys().size()).sum(),+ "the expired row must be a scan candidate");++ svc.deleteTable("TtlTable", REGION);+ svc.createTable("TtlTable",+ List.of(new KeySchemaElement("pk", "HASH")),+ List.of(new AttributeDefinition("pk", "S")),+ 5L, 5L, REGION);+ ObjectNode recreated = mapper.createObjectNode();+ recreated.set("pk", stringAttr("row-0"));+ recreated.set("expireAt", numberAttr(past));+ svc.putItem("TtlTable", recreated, REGION);+ svc.deleteScannedItems(scans);++ ObjectNode key = mapper.createObjectNode();+ key.set("pk", stringAttr("row-0"));+ assertNotNull(svc.getItem("TtlTable", key, REGION),+ "a scan of the deleted table must not delete the recreated table's item");+ }++ @Test+ void aTableDeletedDuringASweepEmitsNoFurtherRemoves() {+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();+ DynamoDbStreamService streamService = mock(DynamoDbStreamService.class);+ DynamoDbService svc = new DynamoDbService(+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"), streamService, null);+ seedTtlTable(svc, 3);+ List<DynamoDbService.ExpiredTableScan> scans = svc.scanExpiredItems();+ assertEquals(3, scans.stream().mapToInt(scan -> scan.itemKeys().size()).sum(),+ "every expired row must be a scan candidate");+ doAnswer(invocation -> {+ svc.deleteTable("TtlTable", REGION);+ return null;+ }).doNothing().when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any());++ svc.deleteScannedItems(scans);++ verify(streamService, times(1)).captureEvent(eq("REMOVE"), any(), any(), any(), any());+ }++ @Test+ void deleteTableWaitsForASweepPersistingTheTable() throws Exception {+ CountDownLatch persisting = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>() {+ @Override+ public void put(String key, Map<String, JsonNode> value) {+ if ("ttl-sweep".equals(Thread.currentThread().getName())) {+ persisting.countDown();+ try {+ release.await(10, TimeUnit.SECONDS);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+ super.put(key, value);+ }+ };+ DynamoDbService svc = new DynamoDbService(+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"));+ seedTtlTable(svc, 1);+ Thread sweep = new Thread(svc::deleteExpiredItems, "ttl-sweep");+ sweep.start();+ assertTrue(persisting.await(10, TimeUnit.SECONDS));++ Thread delete = new Thread(() -> svc.deleteTable("TtlTable", REGION));+ delete.start();+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);+ while (delete.getState() != Thread.State.BLOCKED) {+ if (System.nanoTime() > deadline) {+ fail("DeleteTable did not wait for the sweep persisting the table");+ }+ Thread.onSpinWait();+ }+ release.countDown();+ sweep.join(10_000);+ delete.join(10_000);++ assertEquals(Thread.State.TERMINATED, delete.getState());+ assertTrue(itemStore.get("us-east-1::TtlTable").isEmpty(),+ "DeleteTable must delete the stored items after the sweep wrote them");+ }+// A write landing between scanExpiredItems() and deleteScannedItems() must not be lost.@Testvoid ttlSweep_preserves_item_updated_between_scan_and_delete() throws Exception {diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlServiceTest.javanew file mode 100644index 0000000000..49f9917f3e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlServiceTest.java@@ -0,0 +1,92 @@+package io.github.hectorvent.floci.services.dynamodb;++import org.junit.jupiter.api.Test;++import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.function.Consumer;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;++class DynamoDbTtlServiceTest {++ @Test+ void aFailingSweepDoesNotEscapeTheScheduledTask() {+ DynamoDbService dynamoDbService = mock(DynamoDbService.class);+ doThrow(new IllegalStateException("sweep failed")).when(dynamoDbService).deleteExpiredItems();+ DynamoDbTtlService ttlService = new DynamoDbTtlService(dynamoDbService);++ assertDoesNotThrow(ttlService::sweep);+ }++ @Test+ void aPausedSweepDeletesNothingUntilResumed() {+ DynamoDbService dynamoDbService = mock(DynamoDbService.class);+ DynamoDbTtlService ttlService = new DynamoDbTtlService(dynamoDbService);++ ttlService.pause();+ ttlService.sweep();+ verify(dynamoDbService, never()).deleteExpiredItems();++ ttlService.resume();+ ttlService.sweep();+ verify(dynamoDbService).deleteExpiredItems();+ }++ @Test+ void pauseWaitsForTheSweepInProgress() throws Exception {+ assertWaitsForTheSweepInProgress(DynamoDbTtlService::pause);+ }++ @Test+ void stopWaitsForASweepInProgress() throws Exception {+ assertWaitsForTheSweepInProgress(DynamoDbTtlService::stop);+ }++ /** Also asserts that no sweep deletes anything after {@code call} returns. */+ private static void assertWaitsForTheSweepInProgress(Consumer<DynamoDbTtlService> call) throws Exception {+ DynamoDbService dynamoDbService = mock(DynamoDbService.class);+ CountDownLatch sweeping = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ sweeping.countDown();+ release.await(10, TimeUnit.SECONDS);+ return null;+ }).when(dynamoDbService).deleteExpiredItems();+ DynamoDbTtlService ttlService = new DynamoDbTtlService(dynamoDbService);+ Thread sweep = new Thread(ttlService::sweep);+ sweep.start();+ assertTrue(sweeping.await(10, TimeUnit.SECONDS));++ Thread caller = new Thread(() -> call.accept(ttlService));+ caller.start();+ awaitBlocked(caller);+ release.countDown();+ caller.join(10_000);+ sweep.join(10_000);++ assertEquals(Thread.State.TERMINATED, caller.getState());+ assertEquals(Thread.State.TERMINATED, sweep.getState());+ ttlService.sweep();+ verify(dynamoDbService).deleteExpiredItems();+ }++ /** Observes the calling thread reach the sweep's monitor instead of sleeping to hope for it. */+ private static void awaitBlocked(Thread thread) {+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);+ while (thread.getState() != Thread.State.BLOCKED) {+ if (System.nanoTime() > deadline) {+ fail("the call returned or never reached the sweep's monitor while a sweep was running");+ }+ Thread.onSpinWait();+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendResetIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendResetIntegrationTest.javanew file mode 100644index 0000000000..64b9dffea8--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendResetIntegrationTest.java@@ -0,0 +1,72 @@+package io.github.hectorvent.floci.services.dynamodb.backend;++import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.arc.Arc;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;+import io.restassured.response.ValidatableResponse;+import jakarta.enterprise.context.ApplicationScoped;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import java.util.Set;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertFalse;++/** The emulator lifecycle drives only the selected DynamoDB backend, here one that refuses resets. */+@QuarkusTest+@TestProfile(DynamoDbBackendResetIntegrationTest.RefusingBackendProfile.class)+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class DynamoDbBackendResetIntegrationTest {++ public static final class RefusingBackendProfile implements QuarkusTestProfile {+ @Override+ public Set<Class<?>> getEnabledAlternatives() {+ return Set.of(RefusingDynamoDbBackendLifecycle.class);+ }+ }++ private static final String PARAMETER = "/dynamodb-backend-reset/survives-a-refused-reset";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void startupInitializesNoNativeDatabaseState() {+ assertFalse(Arc.container().getActiveContext(ApplicationScoped.class).getState().getContextualInstances()+ .keySet().stream().anyMatch(bean -> bean.getBeanClass() == DynamoDbService.class));+ }++ @Test+ @Order(2)+ void aRefusedResetClearsNothing() {+ ssm("PutParameter", "{\"Name\": \"" + PARAMETER + "\", \"Value\": \"kept\", \"Type\": \"String\"}")+ .statusCode(200);++ given().when().post("/_floci/state/reset").then().statusCode(409);++ ssm("GetParameter", "{\"Name\": \"" + PARAMETER + "\"}")+ .statusCode(200)+ .body("Parameter.Value", equalTo("kept"));+ }++ private static ValidatableResponse ssm(String action, String body) {+ return given()+ .header("X-Amz-Target", "AmazonSSM." + action)+ .contentType("application/x-amz-json-1.1")+ .body(body)+ .when()+ .post("/")+ .then();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelectionTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelectionTest.javaindex 712518ff15..c4dfdfbe03 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelectionTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbBackendSelectionTest.java@@ -54,6 +54,9 @@ class DynamoDbBackendSelectionTest {@InjectDynamoDbStreamReader streamReader;+ @Inject+ DynamoDbBackendLifecycle lifecycle;+// Collected exactly as ResourceExplorer2Service collects its providers.@InjectInstance<ResourceProvider> providers;@@ -68,6 +71,13 @@ class DynamoDbBackendSelectionTest {assertInstanceOf(NativeDynamoDbBackend.class, backend);assertSame(backend, ClientProxy.unwrap(items));assertSame(backend, ClientProxy.unwrap(tables));+ assertSame(backend, ClientProxy.unwrap(lifecycle));+ }++ @Test+ void startupSchedulesTheTtlSweep() {+ assertTrue(Thread.getAllStackTraces().keySet().stream()+ .anyMatch(thread -> thread.isAlive() && "dynamodb-ttl-sweeper".equals(thread.getName())));}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbNativeBackendTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbNativeBackendTest.javaindex 956a9a7f2b..53c209449c 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbNativeBackendTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbNativeBackendTest.java@@ -7,11 +7,13 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.dynamodb.DynamoDbJsonHandler;import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamsJsonHandler;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlService;import io.github.hectorvent.floci.services.dynamodb.KinesisStreamingForwarder;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbJsonHandler;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbStreamsJsonHandler;@@ -26,6 +28,7 @@ import io.github.hectorvent.floci.services.kinesis.KinesisService;import jakarta.ws.rs.core.Response;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.mockito.InOrder;import java.util.ArrayList;import java.util.List;@@ -39,7 +42,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.inOrder;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;/** The native engine behind the DynamoDB seam, and the thin public handlers in front of it. */@@ -51,20 +56,26 @@ class DynamoDbNativeBackendTest {private static final String TABLE = "orders";private final ObjectMapper mapper = new ObjectMapper();+ private final List<StorageBackend<?, ?>> stores = new ArrayList<>();+ private final KinesisStreamingForwarder forwarder = mock(KinesisStreamingForwarder.class);+ private final DynamoDbTtlService ttl = mock(DynamoDbTtlService.class);private NativeDynamoDbBackend backend;@BeforeEachvoid setUp() {StorageFactory storageFactory = mock(StorageFactory.class);- when(storageFactory.create(anyString(), anyString(), any()))- .thenAnswer(invocation -> AccountAwareStorageBackend.inMemory(ACCOUNT));+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(invocation -> {+ AccountAwareStorageBackend<Object> store = AccountAwareStorageBackend.inMemory(ACCOUNT);+ stores.add(store);+ return store;+ });DynamoDbStreamService streams = new DynamoDbStreamService(mapper, storageFactory);DynamoDbService service = new DynamoDbService(storageFactory, new RegionResolver(REGION, ACCOUNT), streams,- mock(KinesisStreamingForwarder.class), null, mapper, mock(EmulatorConfig.class, RETURNS_DEEP_STUBS));+ forwarder, null, mapper, mock(EmulatorConfig.class, RETURNS_DEEP_STUBS));NativeDynamoDbTableService tables = new NativeDynamoDbTableService(service, streams, mock(KinesisService.class));backend = new NativeDynamoDbBackend(new NativeDynamoDbJsonHandler(service, tables, mapper),new NativeDynamoDbStreamsJsonHandler(streams, service, new RegionResolver(REGION, ACCOUNT), mapper),- service, mapper);+ service, streams, ttl, mapper);}private Reply call(Api api, String action, JsonNode body) throws Exception {@@ -84,6 +95,52 @@ class DynamoDbNativeBackendTest {return mapper.readTree(text);}+ @Test+ void lifecycleDrivesTheTtlSweep() {+ backend.start();+ verify(ttl).start();++ backend.beforeReset();+ verify(ttl).pause();++ backend.afterReset();+ verify(ttl).resume();++ backend.stop();+ verify(ttl).stop();+ }++ @Test+ void stopEndsKinesisForwardingAfterTheTtlSweep() {+ backend.stop();++ InOrder order = inOrder(ttl, forwarder);+ order.verify(ttl).stop();+ order.verify(forwarder).clear();+ }++ @Test+ void resetDropsTheEngineProcessState() throws Exception {+ createTable(",\"StreamSpecification\":{\"StreamEnabled\":true,\"StreamViewType\":\"KEYS_ONLY\"}");+ JsonNode transact = json("""+ {"ClientRequestToken":"token-1",+ "TransactItems":[{"Put":{"TableName":"%s","Item":{"id":{"S":"a"}}}}]}+ """.formatted(TABLE));+ call(Api.DYNAMODB, "TransactWriteItems", transact);++ // An emulator reset wipes Floci storage before it resets the engine.+ stores.forEach(StorageBackend::clear);+ backend.reset();+ createTable("");+ call(Api.DYNAMODB, "TransactWriteItems", transact);++ Reply streams = call(Api.DYNAMODB_STREAMS, "ListStreams", mapper.createObjectNode());+ assertEquals(0, streams.body().path("Streams").size());+ JsonNode key = json("{\"id\":{\"S\":\"a\"}}");+ assertEquals(key, backend.getItem(SCOPE, TABLE, key), "the reused token writes again");+ verify(forwarder).clear();+ }+@Testvoid rawDeleteTableHonoursDeletionProtectionWhileTypedDeleteDoesNot() throws Exception {createTable(",\"DeletionProtectionEnabled\":true");diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/RefusingDynamoDbBackendLifecycle.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/RefusingDynamoDbBackendLifecycle.javanew file mode 100644index 0000000000..902333ed44--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/RefusingDynamoDbBackendLifecycle.java@@ -0,0 +1,39 @@+package io.github.hectorvent.floci.services.dynamodb.backend;++import io.github.hectorvent.floci.core.common.AwsException;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.inject.Alternative;++/** A selected backend that starts nothing and refuses every emulator reset. */+@Alternative+@ApplicationScoped+public class RefusingDynamoDbBackendLifecycle implements DynamoDbBackendLifecycle {++ @Override+ public void start() {+ }++ @Override+ public void checkReset() {+ throw new AwsException("ResourceInUseException", "The DynamoDB backend refuses an emulator reset", 409);+ }++ @Override+ public void beforeReset() {+ throw new IllegalStateException("beforeReset ran after a refused preflight");+ }++ @Override+ public void reset() {+ throw new IllegalStateException("reset ran after a refused preflight");+ }++ @Override+ public void afterReset() {+ throw new IllegalStateException("afterReset ran after a refused preflight");+ }++ @Override+ public void stop() {+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.javaindex 9100047e08..dd5aa083a0 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java@@ -1955,6 +1955,50 @@ class DynamoDbStreamsEventSourcePollerTest {verify(store).saveForAccount(eq(ACCOUNT_ID), any());}+ @Test+ void checkpointIsNotSavedAfterShutdown() throws Exception {+ stubStream("s1");+ EventSourceMapping esm = filterEsm();+ EsmStore store = mock(EsmStore.class);+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);+ AtomicBoolean shutdownDuringInvoke = new AtomicBoolean(true);+ List<List<String>> invocations = recordInvocations(seqs -> {+ if (shutdownDuringInvoke.getAndSet(false)) {+ p.shutdown();+ }+ return new InvokeResult();+ });++ pollOnce(p, esm);++ assertEquals(List.of(List.of("s1")), invocations);+ assertNull(checkpoint(esm));+ verify(store, never()).saveForAccount(anyString(), any());+ }++ @Test+ void aResetEndingAfterShutdownKeepsThePollerQuiesced() throws Exception {+ stubStream("s1");+ EventSourceMapping esm = filterEsm();+ EsmStore store = mock(EsmStore.class);+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);+ AtomicBoolean resetAcrossShutdownDuringInvoke = new AtomicBoolean(true);+ List<List<String>> invocations = recordInvocations(seqs -> {+ if (resetAcrossShutdownDuringInvoke.getAndSet(false)) {+ p.beforeReset();+ p.shutdown();+ p.afterReset();+ }+ return new InvokeResult();+ });++ pollOnce(p, esm);++ assertEquals(List.of(List.of("s1")), invocations);+ assertNull(checkpoint(esm));+ verify(store, never()).saveForAccount(anyString(), any());+ }+@Testvoid pollDuringAResetDoesNotInvokeUntilTheResetEnds() throws Exception {stubStream("s1");diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.javaindex b4bd501c06..272d1d0356 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java@@ -3,9 +3,11 @@ package io.github.hectorvent.floci.services.redshift;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.services.dynamodb.DynamoDbFacade;import io.github.hectorvent.floci.services.dynamodb.DynamoDbService;import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;@@ -17,6 +19,8 @@ import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.redshift.model.Integration;import io.github.hectorvent.floci.services.redshiftdata.RedshiftZeroEtlWriter;import io.quarkus.test.junit.QuarkusTest;+import io.vertx.core.Handler;+import io.vertx.core.Vertx;import jakarta.inject.Inject;import org.junit.jupiter.api.Test;import org.mockito.ArgumentCaptor;@@ -27,19 +31,27 @@ import java.util.Map;import java.util.Set;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.CopyOnWriteArrayList;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyBoolean;+import static org.mockito.ArgumentMatchers.anyLong;import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;import static org.mockito.Mockito.when;@QuarkusTest@@ -260,6 +272,190 @@ class RedshiftDynamoDbZeroEtlConsumerTest {return new DynamoDbStreamReader.Record(sequenceNumber, awsRecord);}+ @Test+ void anEmulatorResetStopsEveryIntegrationTimer() {+ Vertx vertx = mock(Vertx.class);+ when(vertx.setPeriodic(anyLong(), any())).thenReturn(7L);+ RedshiftDynamoDbZeroEtlConsumer consumer = new RedshiftDynamoDbZeroEtlConsumer(vertx,+ new FakeStreamReader(CheckpointLifetime.PROCESS), dynamoDb, mock(RedshiftService.class),+ mock(RedshiftZeroEtlWriter.class), MAPPER, mock(EmulatorConfig.class, RETURNS_DEEP_STUBS));+ consumer.startPolling(streamingIntegration());+ Resettable resettable = consumer;++ resettable.clear();++ verify(vertx).cancelTimer(7L);+ }++ @Test+ void anEmulatorResetWaitsForTheWriteInProgressAndWritesNothingUntilItEnds() throws Exception {+ FakeStreamReader streams = new FakeStreamReader(CheckpointLifetime.PROCESS);+ RedshiftZeroEtlWriter writer = mock(RedshiftZeroEtlWriter.class);+ CountDownLatch writing = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ writing.countDown();+ release.await(10, TimeUnit.SECONDS);+ return null;+ }).doNothing().when(writer).writeBatch(any(), any(), any(), any());+ Integration integration = streamingIntegration();+ List<DynamoDbStreamReader.Record> shard = streams.shard(SHARD_ID, null);+ shard.add(streamRecord("s1"));+ RedshiftDynamoDbZeroEtlConsumer consumer =+ new RedshiftDynamoDbZeroEtlConsumer(streams, dynamoDb, mock(RedshiftService.class), writer);+ Thread poll = new Thread(() -> consumer.pollOnce(integration));+ poll.start();+ assertTrue(writing.await(10, TimeUnit.SECONDS));++ Thread reset = new Thread(consumer::beforeReset);+ reset.start();+ awaitWaiting(reset);+ release.countDown();+ reset.join(10_000);+ poll.join(10_000);+ assertEquals(Thread.State.TERMINATED, reset.getState());++ DynamoDbStreamReader.Record second = streamRecord("s2");+ shard.add(second);+ consumer.pollOnce(integration);+ verify(writer, times(1)).createLandingTable(any(), any(), any());+ verify(writer, times(1)).writeBatch(any(), any(), any(), any());++ consumer.afterReset();+ consumer.pollOnce(integration);+ verify(writer).writeBatch(integration.getAccountId(), "warehouse", "floci_zetl_orders",+ List.of(second.awsRecord()));+ assertEquals(Map.of(SHARD_ID, "s2"), integration.getShardSequenceNumbers());+ }++ @Test+ void aPollSubmittedBeforeAResetIsSkippedEvenWhenItRunsAfterTheReset() {+ FakeStreamReader streams = new FakeStreamReader(CheckpointLifetime.PROCESS);+ RedshiftService redshiftService = mock(RedshiftService.class);+ RedshiftZeroEtlWriter writer = mock(RedshiftZeroEtlWriter.class);+ Integration integration = streamingIntegration();+ DynamoDbStreamReader.Record first = streamRecord("s1");+ streams.shard(SHARD_ID, null).add(first);+ RedshiftDynamoDbZeroEtlConsumer consumer =+ new RedshiftDynamoDbZeroEtlConsumer(streams, dynamoDb, redshiftService, writer);+ Runnable submittedBeforeReset = consumer.pollTask(integration);++ consumer.beforeReset();+ consumer.clear();+ consumer.afterReset();+ submittedBeforeReset.run();++ verifyNoInteractions(writer);+ verify(redshiftService, never()).updateIntegrationRuntime(any(), any(), any(), anyBoolean(), any());++ consumer.pollTask(integration).run();++ verify(writer).writeBatch(integration.getAccountId(), "warehouse", "floci_zetl_orders",+ List.of(first.awsRecord()));+ verify(redshiftService).updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),+ Map.of(SHARD_ID, "s1"), true, null);+ }++ @Test+ void shutdownWaitsForThePollInProgressSoItsCheckpointIsSavedFirst() throws Exception {+ FakeStreamReader streams = new FakeStreamReader(CheckpointLifetime.PROCESS);+ RedshiftService redshiftService = mock(RedshiftService.class);+ RedshiftZeroEtlWriter writer = mock(RedshiftZeroEtlWriter.class);+ List<String> events = new CopyOnWriteArrayList<>();+ CountDownLatch writing = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ writing.countDown();+ release.await(10, TimeUnit.SECONDS);+ return null;+ }).when(writer).writeBatch(any(), any(), any(), any());+ doAnswer(invocation -> events.add("checkpoint saved"))+ .when(redshiftService).updateIntegrationRuntime(any(), any(), any(), anyBoolean(), any());+ Vertx vertx = mock(Vertx.class);+ when(vertx.setPeriodic(anyLong(), any())).thenReturn(7L);+ Integration integration = streamingIntegration();+ streams.shard(SHARD_ID, null).add(streamRecord("s1"));+ RedshiftDynamoDbZeroEtlConsumer consumer = new RedshiftDynamoDbZeroEtlConsumer(vertx, streams, dynamoDb,+ redshiftService, writer, MAPPER, mock(EmulatorConfig.class, RETURNS_DEEP_STUBS));+ consumer.startPolling(integration);+ @SuppressWarnings("unchecked")+ ArgumentCaptor<Handler<Long>> tick = ArgumentCaptor.forClass(Handler.class);+ verify(vertx).setPeriodic(anyLong(), tick.capture());+ Thread shutdown = new Thread(() -> {+ consumer.shutdown();+ events.add("shutdown returned");+ });++ try {+ tick.getValue().handle(7L);+ assertTrue(writing.await(10, TimeUnit.SECONDS), "the tick hands the poll to the poll executor");+ shutdown.start();+ assertEquals(Thread.State.TIMED_WAITING, awaitTimedWaitingOrTerminated(shutdown),+ "shutdown() returned while a poll was still writing");+ release.countDown();+ shutdown.join(10_000);+ } finally {+ release.countDown();+ }++ assertEquals(List.of("checkpoint saved", "shutdown returned"), events);+ verify(redshiftService).updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),+ Map.of(SHARD_ID, "s1"), true, null);+ }++ @Test+ void aTickFromATimerCanceledByAResetDoesNotPoll() {+ FakeStreamReader streams = new FakeStreamReader(CheckpointLifetime.PROCESS);+ RedshiftService redshiftService = mock(RedshiftService.class);+ RedshiftZeroEtlWriter writer = mock(RedshiftZeroEtlWriter.class);+ Vertx vertx = mock(Vertx.class);+ when(vertx.setPeriodic(anyLong(), any())).thenReturn(7L);+ Integration integration = streamingIntegration();+ streams.shard(SHARD_ID, null).add(streamRecord("s1"));+ RedshiftDynamoDbZeroEtlConsumer consumer = new RedshiftDynamoDbZeroEtlConsumer(vertx, streams, dynamoDb,+ redshiftService, writer, MAPPER, mock(EmulatorConfig.class, RETURNS_DEEP_STUBS));+ consumer.startPolling(integration);+ @SuppressWarnings("unchecked")+ ArgumentCaptor<Handler<Long>> tick = ArgumentCaptor.forClass(Handler.class);+ verify(vertx).setPeriodic(anyLong(), tick.capture());++ consumer.beforeReset();+ consumer.clear();+ consumer.afterReset();+ tick.getValue().handle(7L);+ // shutdown() waits for every submitted poll to end, so anything the tick submitted has run.+ consumer.shutdown();++ assertTrue(streams.iteratorRequests.isEmpty(), "the stale tick read the stream");+ verifyNoInteractions(writer);+ verify(redshiftService, never()).updateIntegrationRuntime(any(), any(), any(), anyBoolean(), any());+ }++ /** Observes the thread park in a timed wait, or end, instead of sleeping to hope for either. */+ private static Thread.State awaitTimedWaitingOrTerminated(Thread thread) {+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);+ Thread.State state = thread.getState();+ while (state != Thread.State.TIMED_WAITING && state != Thread.State.TERMINATED) {+ if (System.nanoTime() > deadline) {+ fail("the thread neither parked in a timed wait nor ended");+ }+ Thread.onSpinWait();+ state = thread.getState();+ }+ return state;+ }++ /** Observes the reset thread park on the poll's lock instead of sleeping to hope for it. */+ private static void awaitWaiting(Thread thread) {+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);+ while (thread.getState() != Thread.State.WAITING) {+ if (System.nanoTime() > deadline) {+ fail("beforeReset() returned or never waited while a write was in progress");+ }+ Thread.onSpinWait();+ }+ }+private static Integration streamingIntegration() {Integration integration = integration("orders");integration.setBackfillCompleted(true);