Light, fluffy, and always free - The AWS Local Emulator alternative
feat(redshift-data): support client token idempotency (#6343)
Vũ Thế Huy38c641ec7e37
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.javaindex 6e35a3d2e1..52635353c1 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java@@ -13,6 +13,7 @@ import software.amazon.awssdk.services.redshift.RedshiftClient;import software.amazon.awssdk.services.redshift.model.CreateClusterRequest;import software.amazon.awssdk.services.redshift.model.DeleteClusterRequest;import software.amazon.awssdk.services.redshiftdata.RedshiftDataClient;+import software.amazon.awssdk.services.redshiftdata.model.BatchExecuteStatementRequest;import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementRequest;import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementResponse;import software.amazon.awssdk.services.redshiftdata.model.ExecuteStatementRequest;@@ -74,6 +75,37 @@ class RedshiftDataOperationsTest {}}+ @Test+ @DisplayName("explicit client token retries an INSERT without inserting twice")+ void executeStatementClientTokenRetry() throws Exception {+ run("CREATE TABLE compat_idempotent_single (id int)");+ ExecuteStatementRequest request = ExecuteStatementRequest.builder()+ .clusterIdentifier(clusterId).database(DATABASE).dbUser(USERNAME)+ .clientToken(TestFixtures.uniqueName("single-token"))+ .sql("INSERT INTO compat_idempotent_single VALUES (1)").build();+ String first = data.executeStatement(request).id();+ awaitFinished(first);+ assertThat(data.executeStatement(request).id()).isEqualTo(first);+ String count = run("SELECT COUNT(*) FROM compat_idempotent_single");+ assertThat(data.getStatementResult(r -> r.id(count)).records().get(0).get(0).longValue()).isEqualTo(1L);+ }++ @Test+ @DisplayName("explicit client token retries a batch without repeating its mutations")+ void batchExecuteStatementClientTokenRetry() throws Exception {+ run("CREATE TABLE compat_idempotent_batch (id int)");+ BatchExecuteStatementRequest request = BatchExecuteStatementRequest.builder()+ .clusterIdentifier(clusterId).database(DATABASE).dbUser(USERNAME)+ .clientToken(TestFixtures.uniqueName("batch-token"))+ .sqls("INSERT INTO compat_idempotent_batch VALUES (1)",+ "INSERT INTO compat_idempotent_batch VALUES (2)").build();+ String first = data.batchExecuteStatement(request).id();+ awaitFinished(first);+ assertThat(data.batchExecuteStatement(request).id()).isEqualTo(first);+ String count = run("SELECT COUNT(*) FROM compat_idempotent_batch");+ assertThat(data.getStatementResult(r -> r.id(count)).records().get(0).get(0).longValue()).isEqualTo(2L);+ }+@Test@DisplayName("execute, describe, and get-statement-result round trip")void executeDescribeGetResult() throws Exception {diff --git a/docs/services/redshift-data.md b/docs/services/redshift-data.mdindex f57935cc94..0ff6337ba9 100644--- a/docs/services/redshift-data.md+++ b/docs/services/redshift-data.md@@ -58,6 +58,48 @@ A statement run against a workgroup reports `WorkgroupName` and omits `ClusterId- **`ExecuteSql` and `BatchExecuteSql`** (the deprecated pre-2020 operations) return `ValidationException`.- **Type mapping.** JDBC `BOOLEAN` and `BIT` map to `booleanValue`; integer types to `longValue`; floating-point types to `doubleValue`; `NUMERIC` and `DECIMAL` to `stringValue` (as AWS does); binary types to `blobValue`; everything else, including dates, timestamps, and uuid, to `stringValue`. A SQL `NULL` maps to `isNull`. A result column of type `line`, `json`, or `jsonb` fails the statement with the Redshift error text.+## Client token retries++`ExecuteStatement` and `BatchExecuteStatement` accept an optional `ClientToken`, a case-sensitive+string of 1 to 64 characters. Repeating an admitted request with the same token returns the+original statement `Id` and `CreatedAt` without running SQL or publishing its terminal event+again. This applies while the statement is queued or running and after it finishes, fails or is+cancelled. Omit the token, or use a new one, to submit independent work. AWS SDKs normally generate+the token automatically and keep it for their retries.++Tokens expire 8 hours after first admission. Retries do not extend this deadline. Validation,+reported storage errors and queue-admission failures do not consume the token. An SQL execution failure after+admission does consume it. `WaitTimeSeconds` on a retry can wait for the original execution without+submitting another one.++Request validation and database-target resolution run before admission coordination. Only requests+sharing an account, region, execute operation and token share an admission lock; tokenless requests+use no token lock. The executor reserves queue capacity before writing admission state, so a full+queue cannot create a receipt that reappears after restart. Workers run SQL only after admission+writes succeed. Reset still signals cancellation if clearing either storage backend reports an+error. Expired receipts are removed in one atomic batch per sweep, with concurrently replaced+receipts retained.++Receipts use the configured storage mode. Persistent, hybrid and WAL modes retain them across a+normal restart; memory mode loses them. Interrupted statements keep the existing recovery behavior:+they become `FAILED`, and SQL is never replayed automatically. This does not provide a distributed+transaction between PostgreSQL and Floci storage or an exactly-once guarantee after a host crash.+Persistent and WAL modes use atomic admission writes that report write failures before enqueueing+SQL. Hybrid mode retains its write-behind behavior: disk flush failures are logged while the live+in-memory state remains available.+The token deadline is independent of result retention: if a configured result TTL shorter than+8 hours removes the statement, its receipt still returns the original admission response until+the token expires, without restoring results.++Floci scopes tokens by account, region and execute operation. It validates the request and resolves+the database target before looking up a receipt, including on retries. Within that scope, reusing+a token with changed SQL, parameters or event settings replays the original operation; it does not+change that operation. The precise AWS behavior for changed payloads, different IAM principals,+cross-operation reuse and responses after result eviction has not been verified against a live+AWS cluster. These are explicit Floci policies. See the+[AWS Data API guide](https://docs.aws.amazon.com/redshift/latest/mgmt/data-api.html) for the documented+8-hour token window and SDK retry behavior.+## Statement status events[Redshift Data API status events](https://docs.aws.amazon.com/redshift/latest/mgmt/data-api-monitoring-events.html)diff --git a/pom.xml b/pom.xmlindex 2ad1956285..ccca191e64 100644--- a/pom.xml+++ b/pom.xml@@ -417,6 +417,12 @@<version>${aws.sdk.version}</version><scope>test</scope></dependency>+ <dependency>+ <groupId>software.amazon.awssdk</groupId>+ <artifactId>redshiftdata</artifactId>+ <version>${aws.sdk.version}</version>+ <scope>test</scope>+ </dependency><dependency><groupId>software.amazon.awssdk</groupId><artifactId>url-connection-client</artifactId>diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.javaindex e9c0cd54f7..4f877ccddf 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java@@ -438,6 +438,17 @@ public class AccountAwareStorageBackend<V> implements StorageBackend<String, V>delegate.delete(accountId + "/" + key);}+ @Override+ public void deleteAllAtomically(Set<String> keys) {+ deleteAllAtomicallyForAccounts(Map.of(prefix(), keys));+ }++ public void deleteAllAtomicallyForAccounts(Map<String, Set<String>> keysByAccount) {+ Set<String> rawKeys = new LinkedHashSet<>();+ keysByAccount.forEach((account, keys) -> keys.forEach(key -> rawKeys.add(account + "/" + key)));+ delegate.deleteAllAtomically(rawKeys);+ }+// --- Entry writes, for map-valued stores whose backend persists one entry at a time ---/** Whether the backend persists one map entry at a time; the entry methods require it. */diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.javaindex ff1675c686..5245097e8c 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java@@ -72,6 +72,17 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {dirty.set(true);}+ @Override+ public synchronized void deleteAllAtomically(Set<K> keys) {+ if (keys.isEmpty()) {+ return;+ }+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);+ candidate.keySet().removeAll(Set.copyOf(keys));+ store = candidate;+ dirty.set(true);+ }+@Overridepublic List<V> scan(Predicate<K> keyFilter) {return store.entrySet().stream()diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.javaindex 630e9ee6d4..d645202b74 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java@@ -38,6 +38,16 @@ public class InMemoryStorage<K, V> implements StorageBackend<K, V> {store.remove(key);}+ @Override+ public synchronized void deleteAllAtomically(Set<K> keys) {+ if (keys.isEmpty()) {+ return;+ }+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);+ candidate.keySet().removeAll(Set.copyOf(keys));+ store = candidate;+ }+@Overridepublic List<V> scan(Predicate<K> keyFilter) {List<V> result = new ArrayList<>();diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.javaindex 8df755766e..a23160a19d 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java@@ -85,6 +85,21 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {persistToDisk();}+ @Override+ public synchronized void deleteAllAtomically(Set<K> keys) {+ if (keys.isEmpty()) {+ return;+ }+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);+ candidate.keySet().removeAll(Set.copyOf(keys));+ try {+ StorageFiles.writeAtomically(filePath, tempFile -> objectMapper.writeValue(tempFile.toFile(), candidate));+ } catch (IOException exception) {+ throw new UncheckedIOException("Failed to commit atomic deletions to " + filePath, exception);+ }+ store = candidate;+ }+@Overridepublic List<V> scan(Predicate<K> keyFilter) {return store.entrySet().stream()diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.javaindex c7ba2ab5d8..02c9ceb9fb 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java@@ -39,6 +39,11 @@ public interface StorageBackend<K, V> {void delete(K key);+ /** Remove keys as one storage commit, with the same durability guarantees as atomic puts. */+ default void deleteAllAtomically(Set<K> keys) {+ throw new UnsupportedOperationException("Atomic deletions are not supported by this backend");+ }+/*** Return a new mutable list of values whose keys pass the filter. Callers may sort,* filter, or otherwise mutate the returned list without affecting the underlying store.diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.javaindex 54b295452b..abd1ec7e50 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java@@ -43,6 +43,7 @@ import java.util.stream.Collectors;* PUT: [0x01] [4-byte key length] [key bytes] [4-byte value length] [value bytes]* DELETE: [0x02] [4-byte key length] [key bytes]* BATCH: [0x03] [4-byte map length] [CBOR map bytes]+ * DELETE_BATCH: [0x04] [4-byte array length] [CBOR key array bytes]** Key and value bytes are serialized via Jackson CBOR (compact binary format).* Snapshot files use indented JSON for debuggability.@@ -54,6 +55,7 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {static final byte OP_PUT = 0x01;static final byte OP_DELETE = 0x02;static final byte OP_BATCH = 0x03;+ static final byte OP_DELETE_BATCH = 0x04;private volatile ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();private final Path snapshotPath;@@ -130,6 +132,25 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {} catch (IOException exception) {throw new UncheckedIOException("Failed to encode atomic WAL batch", exception);}+ commitBatch(OP_BATCH, payload, replacements, Set.of());+ }++ @Override+ public void deleteAllAtomically(Set<K> keys) {+ Set<K> deletions = Set.copyOf(keys);+ if (deletions.isEmpty()) {+ return;+ }+ byte[] payload;+ try {+ payload = walMapper.writeValueAsBytes(deletions);+ } catch (IOException exception) {+ throw new UncheckedIOException("Failed to encode atomic WAL deletions", exception);+ }+ commitBatch(OP_DELETE_BATCH, payload, Map.of(), deletions);+ }++ private void commitBatch(byte operation, byte[] payload, Map<K, V> replacements, Set<K> deletions) {compactionLock.writeLock().lock();try {if (walWriter == null) {@@ -137,13 +158,14 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {}ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);candidate.putAll(replacements);+ candidate.keySet().removeAll(deletions);walWriter.flush();FileChannel channel = FileChannel.open(walPath, StandardOpenOption.WRITE);try {long offset = channel.size();channel.position(offset);ByteBuffer record = ByteBuffer.allocate(1 + Integer.BYTES + payload.length);- record.put(OP_BATCH).putInt(payload.length).put(payload).flip();+ record.put(operation).putInt(payload.length).put(payload).flip();try {while (record.hasRemaining()) {channel.write(record);@@ -417,6 +439,14 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {break;}store.putAll(batch);+ } else if (op == OP_DELETE_BATCH) {+ List<K> keys = walMapper.readValue(keyBytes, walMapper.getTypeFactory().constructCollectionType(+ List.class, walMapper.constructType(typeReference.getType()).getKeyType()));+ if (keys == null || keys.stream().anyMatch(key -> key == null)) {+ LOG.warnv("Invalid WAL deletion batch at offset {0} in {1}, stopping replay", consumed, walPath);+ break;+ }+ keys.forEach(store::remove);} else if (op == OP_PUT) {byte[] valueBytes = readFrame(in, fileSize - consumed - recordLength);if (valueBytes == null) {diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutor.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutor.javaindex c3fbcd49b2..f323e94aef 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutor.java@@ -14,12 +14,15 @@ import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.concurrent.ArrayBlockingQueue;+import java.util.concurrent.CountDownLatch;import java.util.concurrent.RejectedExecutionException;import java.util.concurrent.ThreadPoolExecutor;import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.locks.ReentrantReadWriteLock;import java.util.function.BiConsumer;import java.util.function.Consumer;+import java.util.function.Supplier;@ApplicationScopedclass RedshiftDataExecutor implements AutoCloseable {@@ -65,8 +68,9 @@ class RedshiftDataExecutor implements AutoCloseable {}/**- * Admits a statement. {@code admit} runs under the reset read lock, so a reset cannot slip- * between persisting the statement and registering its execution. {@code onFailure} runs when+ * Reserves queue capacity before {@code admit} writes any state. The worker waits for admission+ * to succeed. {@code admit} runs under the reset read lock, so reset cannot split admission.+ * {@code onFailure} runs when* the worker dies with an unexpected exception, so the statement can still reach a terminal state.*/RedshiftDataExecution submit(String account, String region, String id, Runnable admit,@@ -79,7 +83,25 @@ class RedshiftDataExecutor implements AutoCloseable {}String key = account + ":" + id;RedshiftDataExecution execution = new RedshiftDataExecution(generation);+ CountDownLatch ready = new CountDownLatch(1);+ AtomicBoolean admitted = new AtomicBoolean();+ AtomicBoolean claimed = new AtomicBoolean();Runnable task = () -> {+ if (closed && ready.getCount() != 0) {+ return;+ }+ try {+ ready.await();+ } catch (InterruptedException expected) {+ // The admitting thread completes a pending reservation after shutdown interrupts its worker.+ Thread.currentThread().interrupt();+ if (ready.getCount() != 0) {+ return;+ }+ }+ if (!admitted.get() || !claimed.compareAndSet(false, true)) {+ return;+ }try {RequestScopes.runAs(account, region, () -> work.accept(execution));} catch (RuntimeException failure) {@@ -93,7 +115,6 @@ class RedshiftDataExecutor implements AutoCloseable {}};execution.task(task);- admit.run();synchronized (this) {executions.put(key, execution);}@@ -105,12 +126,45 @@ class RedshiftDataExecutor implements AutoCloseable {}throw new AwsException("ActiveStatementsExceededException", "Too many active statements", 400);}+ try {+ admit.run();+ admitted.set(true);+ } catch (RuntimeException | Error failure) {+ workers.remove(task);+ synchronized (this) {+ executions.remove(key, execution);+ }+ execution.completed();+ throw failure;+ } finally {+ boolean finishAfterShutdown;+ synchronized (this) {+ finishAfterShutdown = closed && admitted.get();+ if (finishAfterShutdown) {+ execution.requestCancellation();+ }+ ready.countDown();+ }+ if (finishAfterShutdown) {+ Thread.startVirtualThread(task);+ }+ }return execution;} finally {resetLock.readLock().unlock();}}+ /** Keeps receipt lookup, admission and rejected-admission cleanup in one reset generation. */+ <T> T admitting(Supplier<T> admission) {+ resetLock.readLock().lock();+ try {+ return admission.get();+ } finally {+ resetLock.readLock().unlock();+ }+ }+private static void recover(RedshiftDataExecution execution, RuntimeException failure,BiConsumer<RedshiftDataExecution, RuntimeException> onFailure,String id, String account, String region) {@@ -121,6 +175,10 @@ class RedshiftDataExecutor implements AutoCloseable {}}+ synchronized RedshiftDataExecution execution(String account, String id) {+ return executions.get(account + ":" + id);+ }+boolean cancel(String account, String id) {RedshiftDataExecution execution;synchronized (this) {@@ -173,12 +231,12 @@ class RedshiftDataExecutor implements AutoCloseable {}void clear(Runnable clearStorage) {- List<RedshiftDataExecution> active;+ List<RedshiftDataExecution> active = new ArrayList<>();resetLock.writeLock().lock();try {generation++;synchronized (this) {- active = new ArrayList<>(executions.values());+ active.addAll(executions.values());executions.clear();}for (RedshiftDataExecution execution : active) {@@ -187,13 +245,13 @@ class RedshiftDataExecutor implements AutoCloseable {clearStorage.run();} finally {resetLock.writeLock().unlock();- }- for (RedshiftDataExecution execution : active) {- if (execution.requestCancellation()) {- Thread.startVirtualThread(execution::signalCancellation);- }- if (workers.remove(execution.task())) {- execution.completed();+ for (RedshiftDataExecution execution : active) {+ if (execution.requestCancellation()) {+ Thread.startVirtualThread(execution::signalCancellation);+ }+ if (workers.remove(execution.task())) {+ execution.completed();+ }}}}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStore.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStore.javanew file mode 100644index 0000000000..ec8a92a1c6--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStore.java@@ -0,0 +1,149 @@+package io.github.hectorvent.floci.services.redshiftdata;++import com.fasterxml.jackson.annotation.JsonAutoDetect;+import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.quarkus.runtime.annotations.RegisterForReflection;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;++import java.nio.charset.StandardCharsets;+import java.time.Clock;+import java.time.Duration;+import java.time.Instant;+import java.util.Base64;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.concurrent.locks.ReentrantReadWriteLock;++@ApplicationScoped+class RedshiftDataIdempotencyStore {+ private static final Duration TOKEN_TTL = Duration.ofHours(8);++ private final AccountAwareStorageBackend<Receipt> receipts;+ private final Clock clock;+ private final ReentrantReadWriteLock mutations = new ReentrantReadWriteLock();++ @Inject+ RedshiftDataIdempotencyStore(StorageFactory factory) {+ this(factory.create("redshift-data", "idempotency.json", new TypeReference<Map<String, Receipt>>() { }),+ Clock.systemUTC());+ }++ RedshiftDataIdempotencyStore(AccountAwareStorageBackend<Receipt> receipts, Clock clock) {+ this.receipts = receipts;+ this.clock = clock;+ }++ static String clientToken(JsonNode request) {+ JsonNode token = request.get("ClientToken");+ if (token == null) {+ return null;+ }+ if (!token.isTextual() || token.textValue().isEmpty() || token.textValue().length() > 64) {+ throw new AwsException("ValidationException", "ClientToken must be a string between 1 and 64 characters", 400);+ }+ return token.textValue();+ }++ Receipt find(Scope scope) {+ Receipt receipt = receipts.getForAccount(scope.accountId(), scope.key()).orElse(null);+ if (receipt != null && !clock.instant().isBefore(receipt.expiresAt)) {+ return null;+ }+ return receipt == null ? null : receipt.snapshot();+ }++ void put(Scope scope, ObjectNode response) {+ Receipt receipt = new Receipt();+ receipt.acceptedAt = clock.instant();+ receipt.expiresAt = receipt.acceptedAt.plus(TOKEN_TTL);+ receipt.response = response.deepCopy();+ mutations.readLock().lock();+ try {+ receipts.putAllAtomicallyForAccount(scope.accountId(), Map.of(scope.key(), receipt));+ } finally {+ mutations.readLock().unlock();+ }+ }++ void removeIfStatement(Scope scope, String statementId) {+ mutations.writeLock().lock();+ try {+ Receipt receipt = receipts.getForAccount(scope.accountId(), scope.key()).orElse(null);+ if (receipt != null && statementId.equals(receipt.response.path("Id").asText())) {+ receipts.deleteAllAtomicallyForAccounts(Map.of(scope.accountId(), Set.of(scope.key())));+ }+ } finally {+ mutations.writeLock().unlock();+ }+ }++ void clear() {+ mutations.writeLock().lock();+ try {+ Map<String, Set<String>> keys = new LinkedHashMap<>();+ for (AccountAwareStorageBackend.AccountEntry<Receipt> entry : receipts.scanAllAccountEntries(key -> true)) {+ keys.computeIfAbsent(entry.accountId(), ignored -> new LinkedHashSet<>()).add(entry.key());+ }+ receipts.deleteAllAtomicallyForAccounts(keys);+ } finally {+ mutations.writeLock().unlock();+ }+ }++ void sweep() {+ Instant now = clock.instant();+ List<AccountAwareStorageBackend.AccountEntry<Receipt>> expired = receipts.scanAllAccountEntries(key -> true).stream()+ .filter(entry -> !now.isBefore(entry.value().expiresAt)).toList();+ if (expired.isEmpty()) {+ return;+ }+ mutations.writeLock().lock();+ try {+ Map<String, Set<String>> keys = new LinkedHashMap<>();+ for (AccountAwareStorageBackend.AccountEntry<Receipt> entry : expired) {+ Receipt current = receipts.getForAccount(entry.accountId(), entry.key()).orElse(null);+ if (current == entry.value()) {+ keys.computeIfAbsent(entry.accountId(), ignored -> new LinkedHashSet<>()).add(entry.key());+ }+ }+ receipts.deleteAllAtomicallyForAccounts(keys);+ } finally {+ mutations.writeLock().unlock();+ }+ }++ record Scope(String accountId, String region, String operation, String token) {+ String key() {+ return encode(region) + "." + encode(operation) + "." + encode(token);+ }++ private static String encode(String value) {+ return Base64.getUrlEncoder().withoutPadding().encodeToString(value.getBytes(StandardCharsets.UTF_8));+ }+ }++ @RegisterForReflection+ @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)+ static final class Receipt {+ Instant acceptedAt;+ Instant expiresAt;+ ObjectNode response;++ Receipt snapshot() {+ Receipt copy = new Receipt();+ copy.acceptedAt = acceptedAt;+ copy.expiresAt = expiresAt;+ copy.response = response.deepCopy();+ return copy;+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.javaindex 3c0a8333fe..9919789763 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RequestContext;import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.common.SqlParameterParser.ParsedSql;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.redshift.spectrum.BackendSql;import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumQueryPreparation;@@ -24,6 +25,7 @@ import java.sql.PreparedStatement;import java.sql.ResultSet;import java.sql.ResultSetMetaData;import java.sql.SQLException;+import java.time.Clock;import java.time.Duration;import java.time.Instant;import java.util.ArrayList;@@ -32,6 +34,8 @@ import java.util.Comparator;import java.util.List;import java.util.Map;import java.util.UUID;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.locks.ReentrantLock;import java.util.function.Consumer;import java.util.regex.Pattern;@@ -54,18 +58,22 @@ public class RedshiftDataService implements Resettable {private final RedshiftDataEventPublisher publisher;private final RequestContext requestContext;private final IamService iamService;+ private final RedshiftDataIdempotencyStore idempotency;+ private final ConcurrentHashMap<RedshiftDataIdempotencyStore.Scope, AdmissionLock> admissionLocks = new ConcurrentHashMap<>();RedshiftDataService(RedshiftDataResourceResolver resolver, RedshiftDataConnectionFactory connectionFactory,RedshiftDataStatementStore store, ObjectMapper objectMapper, SpectrumQueryPreparation preparation,RedshiftDataExecutor executor, RedshiftDataEventPublisher publisher) {- this(resolver, connectionFactory, store, objectMapper, preparation, executor, publisher, null, null);+ this(resolver, connectionFactory, store, objectMapper, preparation, executor, publisher, null,+ null, new RedshiftDataIdempotencyStore(AccountAwareStorageBackend.inMemory("000000000000"),+ Clock.systemUTC()));}@Injectpublic RedshiftDataService(RedshiftDataResourceResolver resolver, RedshiftDataConnectionFactory connectionFactory,RedshiftDataStatementStore store, ObjectMapper objectMapper, SpectrumQueryPreparation preparation,RedshiftDataExecutor executor, RedshiftDataEventPublisher publisher,- RequestContext requestContext, IamService iamService) {+ RequestContext requestContext, IamService iamService, RedshiftDataIdempotencyStore idempotency) {this.resolver = resolver;this.connectionFactory = connectionFactory;this.store = store;@@ -75,11 +83,31 @@ public class RedshiftDataService implements Resettable {this.publisher = publisher;this.requestContext = requestContext;this.iamService = iamService;+ this.idempotency = idempotency;}@Overridepublic void clear() {- executor.clear(store::clear);+ executor.clear(() -> {+ RuntimeException failure = null;+ try {+ store.clear();+ } catch (RuntimeException storageFailure) {+ failure = storageFailure;+ }+ try {+ idempotency.clear();+ } catch (RuntimeException receiptFailure) {+ if (failure == null) {+ failure = receiptFailure;+ } else {+ failure.addSuppressed(receiptFailure);+ }+ }+ if (failure != null) {+ throw failure;+ }+ });}// ── ExecuteStatement ────────────────────────────────────────────────────@@ -89,11 +117,58 @@ public class RedshiftDataService implements Resettable {}public ObjectNode executeStatement(JsonNode request, String region, String principal) {+ return execute(request, region, principal, false);+ }++ private ObjectNode execute(JsonNode request, String region, String principal, boolean batch) {+ String token = RedshiftDataIdempotencyStore.clientToken(request);int waitSeconds = waitSeconds(request);- return submit(prepareStatement(request, region), waitSeconds, principal);+ PendingStatement pending = batch ? prepareBatch(request, region) : prepareStatement(request, region);+ RedshiftDataStatementStore.StoredStatement stored = pending.statement();+ RedshiftDataIdempotencyStore.Scope scope = token == null ? null : new RedshiftDataIdempotencyStore.Scope(+ stored.accountId, region, batch ? "BatchExecuteStatement" : "ExecuteStatement", token);+ Admission admission = executor.admitting(() -> scope == null ? submit(pending, principal, null)+ : admitScoped(pending, principal, scope));+ if (waitSeconds > 0 && admission.execution() != null) {+ admission.execution().await(Duration.ofSeconds(waitSeconds));+ RedshiftDataStatementStore.StoredStatement current = store.getForAccount(admission.accountId(), admission.id());+ if (current != null) {+ return executeResponse(current);+ }+ }+ return admission.response();+ }++ private record Admission(String accountId, String id, ObjectNode response, RedshiftDataExecution execution) { }++ private static final class AdmissionLock {+ private final ReentrantLock lock = new ReentrantLock();+ private int users;}- private ObjectNode submit(PendingStatement pending, int waitSeconds, String principal) {+ private Admission admitScoped(PendingStatement pending, String principal, RedshiftDataIdempotencyStore.Scope scope) {+ AdmissionLock scoped = admissionLocks.compute(scope, (key, existing) -> {+ AdmissionLock selected = existing == null ? new AdmissionLock() : existing;+ selected.users++;+ return selected;+ });+ scoped.lock.lock();+ try {+ RedshiftDataIdempotencyStore.Receipt receipt = idempotency.find(scope);+ if (receipt == null) {+ return submit(pending, principal, scope);+ }+ String id = receipt.response.path("Id").asText();+ RedshiftDataStatementStore.StoredStatement current = store.getForAccount(scope.accountId(), id);+ return new Admission(scope.accountId(), id, current == null ? receipt.response : executeResponse(current),+ executor.execution(scope.accountId(), id));+ } finally {+ scoped.lock.unlock();+ admissionLocks.compute(scope, (key, existing) -> --scoped.users == 0 ? null : scoped);+ }+ }++ private Admission submit(PendingStatement pending, String principal, RedshiftDataIdempotencyStore.Scope scope) {RedshiftDataStatementStore.StoredStatement stored = pending.statement();if (principal != null) {stored.principal = principal;@@ -102,7 +177,14 @@ public class RedshiftDataService implements Resettable {RedshiftDataExecution handle;try {handle = executor.submit(stored.accountId, stored.region, stored.id,- () -> store.putForAccount(stored.accountId, stored),+ () -> {+ if (scope == null) {+ store.putForAccount(stored.accountId, stored);+ } else {+ store.admitForAccount(stored.accountId, stored);+ idempotency.put(scope, response);+ }+ },execution -> {try {execution.checkCancellation();@@ -116,18 +198,22 @@ public class RedshiftDataService implements Resettable {complete(stored, execution);},(execution, failure) -> recoverStatement(stored, execution, failure));- } catch (AwsException failure) {- store.deleteForAccount(stored.accountId, stored.id);- throw failure;- }- if (waitSeconds > 0) {- handle.await(Duration.ofSeconds(waitSeconds));- RedshiftDataStatementStore.StoredStatement current = store.getForAccount(stored.accountId, stored.id);- if (current != null) {- return executeResponse(current);+ } catch (RuntimeException failure) {+ try {+ if (scope != null) {+ idempotency.removeIfStatement(scope, stored.id);+ }+ } catch (RuntimeException cleanupFailure) {+ failure.addSuppressed(cleanupFailure);}+ try {+ store.deleteForAccount(stored.accountId, stored.id);+ } catch (RuntimeException cleanupFailure) {+ failure.addSuppressed(cleanupFailure);+ }+ throw failure;}- return response;+ return new Admission(stored.accountId, stored.id, response, handle);}private void complete(RedshiftDataStatementStore.StoredStatement stored, RedshiftDataExecution execution) {@@ -477,8 +563,7 @@ public class RedshiftDataService implements Resettable {}public ObjectNode batchExecuteStatement(JsonNode request, String region, String principal) {- int waitSeconds = waitSeconds(request);- return submit(prepareBatch(request, region), waitSeconds, principal);+ return execute(request, region, principal, true);}private PendingStatement prepareBatch(JsonNode request, String region) {diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataStatementStore.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataStatementStore.javaindex 407b5fffa1..4c0102b0e2 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataStatementStore.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataStatementStore.java@@ -42,22 +42,25 @@ class RedshiftDataStatementStore {private final AccountAwareStorageBackend<StoredStatement> statements;private final Duration ttl;private final Clock clock;+ private final RedshiftDataIdempotencyStore idempotency;// Created by the CDI lifecycle in start(); the test constructor never starts it, so unit// tests that build the store directly drive sweep() by hand and spawn no background thread.private ScheduledExecutorService sweeper;@Inject- RedshiftDataStatementStore(StorageFactory storageFactory, EmulatorConfig config) {+ RedshiftDataStatementStore(StorageFactory storageFactory, EmulatorConfig config, RedshiftDataIdempotencyStore idempotency) {this.statements = storageFactory.create("redshift-data", "statements.json",new TypeReference<Map<String, StoredStatement>>() {});this.ttl = Duration.ofHours(Math.max(1, config.services().redshiftData().resultTtlHours()));this.clock = Clock.systemUTC();+ this.idempotency = idempotency;}RedshiftDataStatementStore(int resultTtlHours, Clock clock) {this.statements = AccountAwareStorageBackend.inMemory("000000000000");this.ttl = Duration.ofHours(Math.max(1, resultTtlHours));this.clock = clock;+ this.idempotency = null;}@PostConstruct@@ -95,10 +98,18 @@ class RedshiftDataStatementStore {statements.putForAccount(accountId, statement.id, statement.snapshot());}+ void admitForAccount(String accountId, StoredStatement statement) {+ statements.putAllAtomicallyForAccount(accountId, Map.of(statement.id, statement.snapshot()));+ }+void deleteForAccount(String accountId, String id) {statements.deleteForAccount(accountId, id);}+ void flush() {+ statements.flush();+ }+Instant expiresAt(Instant createdAt) {return createdAt.plus(ttl);}@@ -120,6 +131,9 @@ class RedshiftDataStatementStore {}void sweep() {+ if (idempotency != null) {+ idempotency.sweep();+ }Instant cutoff = Instant.now(clock).minus(ttl);// The sweep thread has no request context, so it iterates every account's partition// and deletes per account rather than through the current-account view.diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.javaindex 0318a92096..dde1b6abc9 100644--- a/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java@@ -11,6 +11,7 @@ import java.nio.file.Files;import java.nio.file.Path;import java.util.Arrays;import java.util.Map;+import java.util.Set;import java.util.concurrent.CountDownLatch;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;@@ -25,6 +26,33 @@ class AtomicStorageBatchTest {@TempDirPath directory;+ @ParameterizedTest+ @ValueSource(strings = {"memory", "persistent", "hybrid", "wal"})+ void atomicDeletionRemovesSelectedAccountsAndSurvivesRestart(String mode) {+ StorageBackend<String, Integer> backend = backend(mode);+ backend.load();+ try {+ AccountAwareStorageBackend<Integer> accounts = new AccountAwareStorageBackend<>(backend, null, "111111111111");+ accounts.putAllAtomically(Map.of("a", 1, "b", 2));+ accounts.putAllAtomicallyForAccount("222222222222", Map.of("a", 3, "keep", 4));+ accounts.deleteAllAtomicallyForAccounts(Map.of("111111111111", Set.of("a", "b"), "222222222222", Set.of("a")));+ assertEquals(Set.of("222222222222/keep"), backend.keys());+ assertEquals(4, accounts.getForAccount("222222222222", "keep").orElseThrow());+ backend.flush();+ } finally {+ close(backend);+ }+ if (!"memory".equals(mode)) {+ StorageBackend<String, Integer> reopened = backend(mode);+ reopened.load();+ try {+ assertEquals(Set.of("222222222222/keep"), reopened.keys());+ } finally {+ close(reopened);+ }+ }+ }+@ParameterizedTest@ValueSource(strings = {"memory", "persistent", "hybrid", "wal"})void publishesWholeBatchWithAccountIsolation(String mode) {@@ -61,6 +89,7 @@ class AtomicStorageBatchTest {Files.writeString(snapshot.resolve("blocker"), "owned test file");assertThrows(UncheckedIOException.class,() -> store.putAllAtomically(Map.of("source", 1, "destination", 3)));+ assertThrows(UncheckedIOException.class, () -> store.deleteAllAtomically(Set.of("source", "destination")));assertEquals(3, store.get("source").orElseThrow());assertEquals(1, store.get("destination").orElseThrow());PersistentStorage<String, Integer> recovered = new PersistentStorage<>(backup, TYPE);@@ -289,6 +318,7 @@ class AtomicStorageBatchTest {Files.createDirectory(log);assertThrows(UncheckedIOException.class,() -> store.putAllAtomically(Map.of("source", 1, "destination", 3)));+ assertThrows(UncheckedIOException.class, () -> store.deleteAllAtomically(Set.of("source", "destination")));assertEquals(3, store.get("source").orElseThrow());assertEquals(1, store.get("destination").orElseThrow());Files.delete(log);@@ -301,6 +331,39 @@ class AtomicStorageBatchTest {}}+ @Test+ void tornWalDeletionBatchNeverRemovesOnlySomeKeys() throws Exception {+ Path snapshot = directory.resolve("data.json");+ Path log = directory.resolve("data.wal");+ WalStorage<String, Integer> writer = new WalStorage<>(snapshot, log, TYPE, Long.MAX_VALUE);+ writer.load();+ try {+ writer.putAllAtomically(Map.of("a", 1, "b", 2, "keep", 3));+ writer.flush();+ byte[] initial = Files.readAllBytes(snapshot);+ writer.deleteAllAtomically(Set.of("a", "b"));+ byte[] complete = Files.readAllBytes(log);+ for (int length = 0; length <= complete.length; length++) {+ Path replaySnapshot = directory.resolve("replay-" + length + ".json");+ Path replayLog = directory.resolve("replay-" + length + ".wal");+ Files.write(replaySnapshot, initial);+ Files.write(replayLog, Arrays.copyOf(complete, length));+ WalStorage<String, Integer> replay = new WalStorage<>(replaySnapshot, replayLog, TYPE, Long.MAX_VALUE);+ replay.load();+ try {+ boolean deleted = length == complete.length;+ assertEquals(deleted, replay.get("a").isEmpty(), "prefix " + length);+ assertEquals(deleted, replay.get("b").isEmpty(), "prefix " + length);+ assertEquals(3, replay.get("keep").orElseThrow());+ } finally {+ replay.shutdown();+ }+ }+ } finally {+ writer.shutdown();+ }+ }+private StorageBackend<String, Integer> backend(String mode) {return switch (mode) {case "persistent" -> new PersistentStorage<>(directory.resolve("data.json"), TYPE);diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutorTest.javaindex fef015e859..709269a096 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutorTest.java@@ -2,16 +2,130 @@ package io.github.hectorvent.floci.services.redshiftdata;import io.github.hectorvent.floci.core.common.AwsException;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import java.sql.Statement;import java.time.Duration;import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicBoolean;import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.*;class RedshiftDataExecutorTest {+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void shutdownDoesNotWaitForPendingAdmissionAndStillCompletesItAsCancelled(boolean queued) throws Exception {+ try (RedshiftDataExecutor executor = new RedshiftDataExecutor(1, 1, 0);+ ExecutorService callers = Executors.newFixedThreadPool(2)) {+ CountDownLatch busy = new CountDownLatch(1);+ CountDownLatch releaseBusy = new CountDownLatch(1);+ CountDownLatch admitting = new CountDownLatch(1);+ CountDownLatch releaseAdmission = new CountDownLatch(1);+ AtomicBoolean ran = new AtomicBoolean();+ AtomicBoolean cancelled = new AtomicBoolean();+ if (queued) {+ executor.submit("111111111111", "us-east-1", "busy", execution -> {+ busy.countDown();+ try {+ releaseBusy.await(5, TimeUnit.SECONDS);+ } catch (InterruptedException expected) {+ // Shutdown interrupts this occupied worker before it can dequeue the reservation.+ Thread.currentThread().interrupt();+ }+ });+ assertTrue(busy.await(5, TimeUnit.SECONDS));+ }+ Future<RedshiftDataExecution> submission = callers.submit(() -> executor.submit(+ "111111111111", "us-east-1", "pending", () -> {+ admitting.countDown();+ await(releaseAdmission);+ }, execution -> {+ ran.set(true);+ cancelled.set(execution.isCancellationRequested());+ }, (execution, failure) -> { }));+ assertTrue(admitting.await(5, TimeUnit.SECONDS));+ Future<?> shutdown = callers.submit(executor::close);+ try {+ shutdown.get(1, TimeUnit.SECONDS);+ } finally {+ releaseAdmission.countDown();+ releaseBusy.countDown();+ }+ RedshiftDataExecution pending = submission.get(5, TimeUnit.SECONDS);+ pending.await(Duration.ofSeconds(2));+ assertTrue(pending.isDone(), "late admission must not strand a SUBMITTED execution");+ assertTrue(ran.get());+ assertTrue(cancelled.get());+ }+ }++ @Test+ void rejectedQueueDoesNotRunAdmissionWrites() throws Exception {+ try (RedshiftDataExecutor executor = new RedshiftDataExecutor(1, 1, 1)) {+ CountDownLatch started = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ try {+ executor.submit("111111111111", "us-east-1", "running", execution -> {+ started.countDown();+ await(release);+ });+ assertTrue(started.await(5, TimeUnit.SECONDS));+ executor.submit("111111111111", "us-east-1", "queued", execution -> { });+ AtomicBoolean written = new AtomicBoolean();+ assertThrows(AwsException.class, () -> executor.submit("111111111111", "us-east-1", "rejected",+ () -> written.set(true), execution -> { }, (execution, failure) -> { }));+ assertFalse(written.get(), "a rejected queue entry must not leave durable admission state");+ } finally {+ release.countDown();+ }+ }+ }++ @Test+ void failingStorageResetStillSignalsJdbcCancellationAndCompletesQueuedWork() throws Exception {+ try (RedshiftDataExecutor executor = new RedshiftDataExecutor(1, 1, 1)) {+ Statement jdbc = mock(Statement.class);+ CountDownLatch started = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ CountDownLatch cancelled = new CountDownLatch(1);+ doAnswer(invocation -> {+ cancelled.countDown();+ return null;+ }).when(jdbc).cancel();+ try {+ executor.submit("111111111111", "us-east-1", "running", execution -> {+ try {+ execution.track(jdbc);+ } catch (Exception failure) {+ throw new AssertionError(failure);+ }+ started.countDown();+ await(release);+ });+ assertTrue(started.await(5, TimeUnit.SECONDS));+ AtomicBoolean queuedRan = new AtomicBoolean();+ RedshiftDataExecution queued = executor.submit("111111111111", "us-east-1", "queued",+ execution -> queuedRan.set(true));+ IllegalStateException storageFailure = new IllegalStateException("receipt clear failed");+ assertSame(storageFailure, assertThrows(IllegalStateException.class, () -> executor.clear(() -> {+ throw storageFailure;+ })));+ assertTrue(cancelled.await(2, TimeUnit.SECONDS), "reset failure must not skip JDBC cancellation");+ assertTrue(queued.isDone());+ assertFalse(queuedRan.get());+ } finally {+ release.countDown();+ }+ }+ }+@Testvoid admissionReturnsWhileSqlIsRunningAndRejectsOverflow() throws Exception {try (RedshiftDataExecutor executor = new RedshiftDataExecutor(1, 1, 1)) {diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyIntegrationTest.javanew file mode 100644index 0000000000..0221425fa9--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyIntegrationTest.java@@ -0,0 +1,146 @@+package io.github.hectorvent.floci.services.redshiftdata;++import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.InjectMock;+import io.quarkus.test.common.http.TestHTTPResource;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.h2.Driver;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.redshiftdata.RedshiftDataClient;+import software.amazon.awssdk.services.redshiftdata.model.BatchExecuteStatementRequest;+import software.amazon.awssdk.services.redshiftdata.model.ExecuteStatementRequest;+import software.amazon.awssdk.services.redshiftdata.model.ExecuteStatementResponse;+import software.amazon.awssdk.services.redshiftdata.model.ResourceNotFoundException;+import software.amazon.awssdk.services.redshiftdata.model.StatusString;++import java.net.URI;+import java.sql.Connection;+import java.sql.DriverManager;+import java.sql.Statement;+import java.time.Duration;+import java.util.UUID;++import static org.awaitility.Awaitility.await;+import static org.hamcrest.Matchers.containsString;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.when;++@QuarkusTest+class RedshiftDataIdempotencyIntegrationTest {+ @TestHTTPResource("/")+ URI endpoint;++ @InjectMock+ RedshiftDataResourceResolver resolver;+ @InjectMock+ RedshiftDataConnectionFactory factory;+ @Inject+ RedshiftDataService service;+ @Inject+ RegionResolver regions;++ private RedshiftDataClient data;++ @BeforeEach+ void setup() throws Exception {+ Driver.load();+ String jdbcUrl = "jdbc:h2:mem:" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1";+ try (Connection connection = DriverManager.getConnection(jdbcUrl); Statement statement = connection.createStatement()) {+ statement.execute("CREATE TABLE mutations (id INT)");+ }+ when(factory.open(any())).thenAnswer(invocation -> DriverManager.getConnection(jdbcUrl));+ when(resolver.resolve(any(), any())).thenAnswer(invocation -> {+ String arn = AwsArnUtils.Arn.of("redshift", invocation.getArgument(1), regions.getAccountId(), "cluster:wh").toString();+ return new RedshiftDataResourceResolver.DatabaseTarget(arn, "localhost", 5439, "dev", "admin", "password");+ });+ data = RedshiftDataClient.builder().endpointOverride(endpoint).region(Region.US_EAST_1)+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test"))).build();+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @AfterEach+ void cleanup() {+ service.clear();+ if (data != null) {+ data.close();+ }+ }++ @Test+ void sdkRetriesSingleAndWorkgroupStatementsWithoutRepeatingSql() {+ for (boolean serverless : new boolean[] {false, true}) {+ ExecuteStatementRequest.Builder builder = ExecuteStatementRequest.builder().database("dev")+ .clientToken(UUID.randomUUID().toString()).sql("INSERT INTO mutations VALUES (1)");+ if (serverless) {+ builder.workgroupName("analytics");+ } else {+ builder.clusterIdentifier("wh").dbUser("admin");+ }+ ExecuteStatementRequest request = builder.build();+ ExecuteStatementResponse first = data.executeStatement(request);+ ExecuteStatementResponse retry = data.executeStatement(request);+ assertEquals(first.id(), retry.id());+ assertEquals(first.createdAt(), retry.createdAt());+ finished(first.id());+ }+ assertEquals(2L, count());+ }++ @Test+ void sdkRetriesBatchWithoutRepeatingEitherMutation() {+ BatchExecuteStatementRequest request = BatchExecuteStatementRequest.builder()+ .clusterIdentifier("wh").dbUser("admin").database("dev")+ .clientToken(UUID.randomUUID().toString())+ .sqls("INSERT INTO mutations VALUES (1)", "INSERT INTO mutations VALUES (2)").build();+ String id = data.batchExecuteStatement(request).id();+ assertEquals(id, data.batchExecuteStatement(request).id());+ finished(id);+ assertEquals(2L, count());+ }++ @Test+ void malformedTokenReturnsAnAwsJsonValidationError() {+ RestAssuredJsonUtils.awsAction("RedshiftData", "ExecuteStatement", """+ {"ClusterIdentifier":"wh","DbUser":"admin","Database":"dev","Sql":"SELECT 1","ClientToken":12}+ """).then().statusCode(400).body("__type", containsString("ValidationException"));+ }++ @Test+ void sdkTokenReceiptsAreIsolatedByTheSigningAccount() {+ ExecuteStatementRequest request = ExecuteStatementRequest.builder().clusterIdentifier("wh").dbUser("admin")+ .database("dev").clientToken(UUID.randomUUID().toString()).sql("SELECT 1").build();+ String first = data.executeStatement(request).id();+ try (RedshiftDataClient other = RedshiftDataClient.builder().endpointOverride(endpoint).region(Region.US_EAST_1)+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("111111111111", "test"))).build()) {+ String second = other.executeStatement(request).id();+ assertNotEquals(first, second);+ assertEquals(second, other.executeStatement(request).id());+ assertEquals(first, data.executeStatement(request).id());+ assertThrows(ResourceNotFoundException.class, () -> data.describeStatement(r -> r.id(second)));+ }+ finished(first);+ }++ private long count() {+ String id = data.executeStatement(r -> r.clusterIdentifier("wh").dbUser("admin").database("dev")+ .sql("SELECT COUNT(*) FROM mutations")).id();+ finished(id);+ return data.getStatementResult(r -> r.id(id)).records().getFirst().getFirst().longValue();+ }++ private void finished(String id) {+ await().atMost(Duration.ofSeconds(10)).untilAsserted(() ->+ assertEquals(StatusString.FINISHED, data.describeStatement(r -> r.id(id)).status()));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyServiceTest.javanew file mode 100644index 0000000000..1945666e56--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyServiceTest.java@@ -0,0 +1,607 @@+package io.github.hectorvent.floci.services.redshiftdata;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.PersistentStorage;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import org.h2.Driver;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.UncheckedIOException;+import java.nio.file.Files;+import java.nio.file.Path;+import java.sql.Connection;+import java.sql.DriverManager;+import java.sql.ResultSet;+import java.sql.Statement;+import java.time.Clock;+import java.time.Duration;+import java.time.Instant;+import java.time.ZoneId;+import java.time.ZoneOffset;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.UUID;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.concurrent.atomic.AtomicInteger;++import static org.awaitility.Awaitility.await;+import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.*;++class RedshiftDataIdempotencyServiceTest {++ @Test+ void slowReceiptWriteDoesNotBlockOtherTokensOrTokenlessRequests() throws Exception {+ try (Fixture fixture = new Fixture(); ExecutorService callers = Executors.newFixedThreadPool(3)) {+ CountDownLatch writing = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ RedshiftDataIdempotencyStore.Scope scope = invocation.getArgument(0);+ if (scope.token().equals("slow")) {+ writing.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ }+ return invocation.callRealMethod();+ }).when(fixture.receipts).put(any(), any());+ Future<ObjectNode> slow = callers.submit(() -> fixture.service.executeStatement(+ fixture.single("slow", "SELECT 1"), "us-east-1"));+ try {+ assertTrue(writing.await(5, TimeUnit.SECONDS));+ Future<ObjectNode> independent = callers.submit(() -> fixture.service.executeStatement(+ fixture.single("independent", "SELECT 1"), "us-east-1"));+ Future<ObjectNode> tokenless = callers.submit(() -> {+ ObjectNode request = fixture.single("unused", "SELECT 1");+ request.remove("ClientToken");+ return fixture.service.executeStatement(request, "us-east-1");+ });+ assertFalse(independent.get(2, TimeUnit.SECONDS).path("Id").asText().isEmpty());+ assertFalse(tokenless.get(2, TimeUnit.SECONDS).path("Id").asText().isEmpty());+ } finally {+ release.countDown();+ }+ slow.get(5, TimeUnit.SECONDS);+ }+ }++ @Test+ void rejectedQueueCannotLeaveAReceiptAfterCleanupFailureAndRestart(@TempDir Path directory) throws Exception {+ try (Fixture fixture = new Fixture()) {+ Path receiptFile = directory.resolve("idempotency.json");+ RedshiftDataIdempotencyStore receipts = spy(persistentReceipts(receiptFile, fixture.clock));+ RedshiftDataService service = fixture.service(fixture.statements, receipts);+ doThrow(new IllegalStateException("receipt deletion unavailable")).when(receipts).removeIfStatement(any(), any());+ CountDownLatch started = new CountDownLatch(2);+ CountDownLatch release = new CountDownLatch(1);+ try {+ for (int i = 0; i < 2; i++) {+ fixture.executor.submit("000000000000", "us-east-1", "running-" + i, execution -> {+ started.countDown();+ try {+ assertTrue(release.await(5, TimeUnit.SECONDS));+ } catch (InterruptedException failure) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(failure);+ }+ });+ }+ assertTrue(started.await(5, TimeUnit.SECONDS));+ for (int i = 0; i < 16; i++) {+ fixture.executor.submit("000000000000", "us-east-1", "queued-" + i, execution -> { });+ }+ assertThrows(AwsException.class, () -> service.executeStatement(+ fixture.single("rejected-persistent", "INSERT INTO mutations VALUES (1)"), "us-east-1"));+ RedshiftDataIdempotencyStore reopened = persistentReceipts(receiptFile, fixture.clock);+ assertNull(reopened.find(new RedshiftDataIdempotencyStore.Scope("000000000000", "us-east-1",+ "ExecuteStatement", "rejected-persistent")));+ verifyNoInteractions(fixture.factory);+ } finally {+ release.countDown();+ }+ }+ }++ @Test+ void executeRetryReturnsOriginalIdentityAndMutatesOnce() throws Exception {+ try (Fixture fixture = new Fixture()) {+ ObjectNode request = fixture.single("insert-token", "INSERT INTO mutations VALUES (1)");+ ObjectNode first = fixture.service.executeStatement(request, "us-east-1");+ ObjectNode retry = fixture.service.executeStatement(request, "us-east-1");+ assertEquals(first.path("Id"), retry.path("Id"));+ assertEquals(first.path("CreatedAt"), retry.path("CreatedAt"));+ assertEquals(1, fixture.count());+ verify(fixture.factory, times(1)).open(any());+ }+ }++ @Test+ void batchRetryMutatesOnceAndPublishesOneParentEvent() throws Exception {+ try (Fixture fixture = new Fixture()) {+ ObjectNode request = fixture.mapper.createObjectNode().put("ClientToken", "batch-token")+ .put("WaitTimeSeconds", 5).put("WithEvent", true);+ request.putArray("Sqls").add("INSERT INTO mutations VALUES (1)")+ .add("INSERT INTO mutations VALUES (2)");+ ObjectNode first = fixture.service.batchExecuteStatement(request, "us-east-1");+ ObjectNode retry = fixture.service.batchExecuteStatement(request, "us-east-1");+ assertEquals(first.path("Id"), retry.path("Id"));+ assertEquals(2, fixture.count());+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ verify(fixture.publisher, times(1)).publish(any()));+ verify(fixture.factory, times(1)).open(any());+ }+ }++ @Test+ void absentDifferentAndDifferentlyCasedTokensAreIndependent() throws Exception {+ try (Fixture fixture = new Fixture()) {+ List<String> ids = new ArrayList<>();+ for (String token : List.of("one", "One", "two", "", "")) {+ ObjectNode request = fixture.single(token, "INSERT INTO mutations VALUES (1)");+ if (token.isEmpty()) {+ request.remove("ClientToken");+ }+ ids.add(fixture.service.executeStatement(request, "us-east-1").path("Id").asText());+ }+ assertEquals(5, ids.stream().distinct().count());+ assertEquals(5, fixture.count());+ }+ }++ @Test+ void suppliedMalformedTokensAreRejectedBeforeAdmission() throws Exception {+ try (Fixture fixture = new Fixture()) {+ for (String token : List.of("", "x".repeat(65))) {+ AwsException failure = assertThrows(AwsException.class, () -> fixture.service.executeStatement(+ fixture.single(token, "SELECT 1"), "us-east-1"));+ assertEquals("ValidationException", failure.getErrorCode());+ }+ ObjectNode numeric = fixture.single("token", "SELECT 1").put("ClientToken", 123);+ assertThrows(AwsException.class, () -> fixture.service.batchExecuteStatement(numeric, "us-east-1"));+ ObjectNode nullToken = fixture.single("token", "SELECT 1").putNull("ClientToken");+ assertThrows(AwsException.class, () -> fixture.service.executeStatement(nullToken, "us-east-1"));+ verifyNoInteractions(fixture.factory);+ }+ }++ @Test+ void concurrentRetriesOfRunningWorkAdmitOneStatement() throws Exception {+ try (Fixture fixture = new Fixture(); ExecutorService clients = Executors.newFixedThreadPool(8)) {+ CountDownLatch opened = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ when(fixture.factory.open(any())).thenAnswer(invocation -> {+ opened.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ return DriverManager.getConnection(fixture.jdbcUrl);+ });+ ObjectNode request = fixture.single("parallel", "INSERT INTO mutations VALUES (1)").put("WithEvent", true);+ request.remove("WaitTimeSeconds");+ try {+ List<Future<String>> replies = new ArrayList<>();+ for (int i = 0; i < 8; i++) {+ replies.add(clients.submit(() -> fixture.service.executeStatement(request, "us-east-1")+ .path("Id").asText()));+ }+ assertTrue(opened.await(5, TimeUnit.SECONDS));+ List<String> ids = new ArrayList<>();+ for (Future<String> reply : replies) {+ ids.add(reply.get(5, TimeUnit.SECONDS));+ }+ assertEquals(1, ids.stream().distinct().count());+ } finally {+ release.countDown();+ }+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertEquals(1, fixture.count()));+ verify(fixture.factory, times(1)).open(any());+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> verify(fixture.publisher, times(1)).publish(any()));+ }+ }++ @Test+ void failedStatementReplaysUntilReset() throws Exception {+ try (Fixture fixture = new Fixture()) {+ ObjectNode request = fixture.single("failed", "SELECT * FROM missing_table");+ String id = fixture.service.executeStatement(request, "us-east-1").path("Id").asText();+ assertEquals(RedshiftDataStatementStore.Status.FAILED, fixture.statements.get(id).status);+ assertEquals(id, fixture.service.executeStatement(request, "us-east-1").path("Id").asText());+ fixture.service.clear();+ assertNotEquals(id, fixture.service.executeStatement(request, "us-east-1").path("Id").asText());+ verify(fixture.factory, times(2)).open(any());+ }+ }++ @Test+ void tokenScopeSeparatesRegionsAndExecuteOperations() throws Exception {+ try (Fixture fixture = new Fixture()) {+ String first = fixture.service.executeStatement(fixture.single("shared", "SELECT 1"), "us-east-1")+ .path("Id").asText();+ String otherRegion = fixture.service.executeStatement(fixture.single("shared", "SELECT 1"), "eu-west-1")+ .path("Id").asText();+ ObjectNode batch = fixture.mapper.createObjectNode().put("ClientToken", "shared").put("WaitTimeSeconds", 5);+ batch.putArray("Sqls").add("SELECT 1");+ String otherOperation = fixture.service.batchExecuteStatement(batch, "us-east-1").path("Id").asText();+ assertEquals(3, List.of(first, otherRegion, otherOperation).stream().distinct().count());+ }+ }++ @Test+ void sameTokenReplaysOriginalPayloadAndDoesNotChangeEventPreference() throws Exception {+ try (Fixture fixture = new Fixture()) {+ ObjectNode first = fixture.service.executeStatement(+ fixture.single("reused", "INSERT INTO mutations VALUES (1)"), "us-east-1");+ ObjectNode retry = fixture.service.executeStatement(+ fixture.single("reused", "INSERT INTO mutations VALUES (2)").put("WithEvent", true), "us-east-1");+ assertEquals(first.path("Id"), retry.path("Id"));+ assertEquals(1, fixture.count());+ verifyNoInteractions(fixture.publisher);+ }+ }++ @Test+ void replayStillValidatesDatabaseAccess() throws Exception {+ try (Fixture fixture = new Fixture()) {+ ObjectNode request = fixture.single("access", "INSERT INTO mutations VALUES (1)");+ fixture.service.executeStatement(request, "us-east-1");+ when(fixture.resolver.resolve(any(), any())).thenThrow(new AwsException("ValidationException", "Access revoked", 400));+ assertThrows(AwsException.class, () -> fixture.service.executeStatement(request, "us-east-1"));+ verify(fixture.factory, times(1)).open(any());+ }+ }++ @Test+ void expiredTokenAdmitsNewWorkAndShortResultTtlDoesNotLoseTheReceipt() throws Exception {+ try (Fixture fixture = new Fixture()) {+ ObjectNode request = fixture.single("retention", "INSERT INTO mutations VALUES (1)");+ ObjectNode first = fixture.service.executeStatement(request, "us-east-1");+ fixture.clock.advance(Duration.ofHours(2));+ fixture.statements.sweep();+ assertNull(fixture.statements.get(first.path("Id").asText()));+ assertEquals(first, fixture.service.executeStatement(request, "us-east-1"));+ assertEquals(1, fixture.count());+ fixture.clock.advance(Duration.ofHours(6));+ assertNotEquals(first.path("Id"), fixture.service.executeStatement(request, "us-east-1").path("Id"));+ assertEquals(2, fixture.count());+ }+ }++ @Test+ void rejectedAdmissionAndStorageFailuresLeaveTokenReusable() throws Exception {+ try (Fixture fixture = new Fixture()) {+ ObjectNode request = fixture.single("retry-error", "INSERT INTO mutations VALUES (1)");+ doThrow(new IllegalStateException("receipt write failed")).doCallRealMethod()+ .when(fixture.receipts).put(any(), any());+ assertThrows(IllegalStateException.class, () -> fixture.service.executeStatement(request, "us-east-1"));+ assertTrue(fixture.statements.values().isEmpty());+ verifyNoInteractions(fixture.factory);+ fixture.service.executeStatement(request, "us-east-1");+ assertEquals(1, fixture.count());+ }+ }++ @Test+ void statementWriteFailureDoesNotConsumeToken() throws Exception {+ try (Fixture fixture = new Fixture()) {+ ObjectNode request = fixture.single("statement-write", "INSERT INTO mutations VALUES (1)");+ doThrow(new IllegalStateException("statement write failed")).doCallRealMethod()+ .when(fixture.statements).admitForAccount(any(), any());+ assertThrows(IllegalStateException.class, () -> fixture.service.executeStatement(request, "us-east-1"));+ verifyNoInteractions(fixture.factory);+ fixture.service.executeStatement(request, "us-east-1");+ assertEquals(1, fixture.count());+ }+ }++ @Test+ void queueFullReplaysAcceptedTokenAndRejectedTokenCanBeRetried() throws Exception {+ try (Fixture fixture = new Fixture()) {+ fixture.service.executeStatement(fixture.single("accepted", "INSERT INTO mutations VALUES (1)"), "us-east-1");+ CountDownLatch started = new CountDownLatch(2);+ CountDownLatch release = new CountDownLatch(1);+ try {+ for (int i = 0; i < 2; i++) {+ fixture.executor.submit("000000000000", "us-east-1", "blocker-" + i, execution -> {+ started.countDown();+ try {+ assertTrue(release.await(5, TimeUnit.SECONDS));+ } catch (InterruptedException failure) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(failure);+ }+ });+ }+ assertTrue(started.await(5, TimeUnit.SECONDS));+ ObjectNode queued = fixture.single("queued", "INSERT INTO mutations VALUES (2)");+ queued.remove("WaitTimeSeconds");+ for (int i = 0; i < 16; i++) {+ fixture.service.executeStatement(queued.deepCopy().put("ClientToken", "queued-" + i), "us-east-1");+ }+ ObjectNode rejected = fixture.single("rejected", "INSERT INTO mutations VALUES (3)");+ rejected.remove("WaitTimeSeconds");+ AwsException failure = assertThrows(AwsException.class,+ () -> fixture.service.executeStatement(rejected, "us-east-1"));+ assertEquals("ActiveStatementsExceededException", failure.getErrorCode());+ fixture.service.executeStatement(fixture.single("accepted", "INSERT INTO mutations VALUES (1)"), "us-east-1");+ } finally {+ release.countDown();+ }+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertEquals(17, fixture.count()));+ fixture.service.executeStatement(fixture.single("rejected", "INSERT INTO mutations VALUES (3)"), "us-east-1");+ assertEquals(18, fixture.count());+ }+ }++ @Test+ void restartRetainsReceiptAndFailsInterruptedWorkWithoutReexecutingSql(@TempDir Path directory) throws Exception {+ try (Fixture fixture = new Fixture()) {+ Path receiptFile = directory.resolve("idempotency.json");+ Path statementFile = directory.resolve("statements.json");+ RedshiftDataIdempotencyStore originalReceipts = persistentReceipts(receiptFile, fixture.clock);+ RedshiftDataStatementStore originalStatements = persistentStatements(statementFile, originalReceipts);+ RedshiftDataService firstService = fixture.service(originalStatements, originalReceipts);+ ObjectNode request = fixture.single("restart", "INSERT INTO mutations VALUES (1)");+ String id = firstService.executeStatement(request, "us-east-1").path("Id").asText();+ RedshiftDataStatementStore.StoredStatement interrupted = originalStatements.get(id);+ interrupted.status = RedshiftDataStatementStore.Status.STARTED;+ originalStatements.put(interrupted);+ originalStatements.flush();++ RedshiftDataIdempotencyStore reopenedReceipts = persistentReceipts(receiptFile, fixture.clock);+ RedshiftDataStatementStore reopenedStatements = persistentStatements(statementFile, reopenedReceipts);+ reopenedStatements.start();+ try {+ assertEquals(RedshiftDataStatementStore.Status.FAILED, reopenedStatements.get(id).status);+ RedshiftDataService reopenedService = fixture.service(reopenedStatements, reopenedReceipts);+ assertEquals(id, reopenedService.executeStatement(request, "us-east-1").path("Id").asText());+ assertEquals(1, fixture.count());+ verify(fixture.factory, times(1)).open(any());+ verifyNoInteractions(fixture.publisher);+ } finally {+ reopenedStatements.stop();+ }+ }+ }++ @Test+ void persistentStatementWriteFailureCannotAdmitSqlOrPublishAReceipt(@TempDir Path directory) throws Exception {+ try (Fixture fixture = new Fixture()) {+ Path statementFile = directory.resolve("statements.json");+ RedshiftDataStatementStore statements = persistentStatements(statementFile, fixture.receipts);+ Files.createDirectory(statementFile);+ Files.writeString(statementFile.resolve("blocked"), "cannot replace a nonempty directory");+ RedshiftDataService service = fixture.service(statements, fixture.receipts);+ assertThrows(UncheckedIOException.class,+ () -> service.executeStatement(fixture.single("disk-failure", "INSERT INTO mutations VALUES (1)"), "us-east-1"));+ verifyNoInteractions(fixture.factory);+ verify(fixture.receipts, never()).put(any(), any());+ assertTrue(statements.values().isEmpty());+ }+ }++ @Test+ void resetWaitsForAdmissionAndClearsItsReceipt() throws Exception {+ try (Fixture fixture = new Fixture(); ExecutorService callers = Executors.newFixedThreadPool(2)) {+ CountDownLatch writing = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ writing.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ return invocation.callRealMethod();+ }).when(fixture.receipts).put(any(), any());+ ObjectNode request = fixture.single("reset-race", "INSERT INTO mutations VALUES (1)");+ request.remove("WaitTimeSeconds");+ Future<ObjectNode> admitted = callers.submit(() -> fixture.service.executeStatement(request, "us-east-1"));+ assertTrue(writing.await(5, TimeUnit.SECONDS));+ Future<?> reset = callers.submit(fixture.service::clear);+ try {+ assertThrows(TimeoutException.class, () -> reset.get(100, TimeUnit.MILLISECONDS));+ } finally {+ release.countDown();+ }+ String originalId = admitted.get(5, TimeUnit.SECONDS).path("Id").asText();+ reset.get(5, TimeUnit.SECONDS);+ assertTrue(fixture.statements.values().isEmpty());+ assertNotEquals(originalId, fixture.service.executeStatement(request, "us-east-1").path("Id").asText());+ }+ }++ @Test+ void cancelledQueuedStatementReplaysWithoutOpeningAConnection() throws Exception {+ try (Fixture fixture = new Fixture()) {+ CountDownLatch started = new CountDownLatch(2);+ CountDownLatch release = new CountDownLatch(1);+ try {+ for (int i = 0; i < 2; i++) {+ fixture.executor.submit("000000000000", "us-east-1", "blocker-" + i, execution -> {+ started.countDown();+ try {+ assertTrue(release.await(5, TimeUnit.SECONDS));+ } catch (InterruptedException failure) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(failure);+ }+ });+ }+ assertTrue(started.await(5, TimeUnit.SECONDS));+ ObjectNode request = fixture.single("cancelled", "INSERT INTO mutations VALUES (1)");+ request.remove("WaitTimeSeconds");+ String id = fixture.service.executeStatement(request, "us-east-1").path("Id").asText();+ assertTrue(fixture.service.cancelStatement(fixture.mapper.createObjectNode().put("Id", id))+ .path("Status").asBoolean());+ assertEquals(RedshiftDataStatementStore.Status.ABORTED, fixture.statements.get(id).status);+ assertEquals(id, fixture.service.executeStatement(request, "us-east-1").path("Id").asText());+ verifyNoInteractions(fixture.factory);+ } finally {+ release.countDown();+ }+ }+ }++ @Test+ void expiryDuringExecutionDoesNotLetOldCompletionReplaceTheNewReceipt() throws Exception {+ try (Fixture fixture = new Fixture()) {+ CountDownLatch opened = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ AtomicInteger connections = new AtomicInteger();+ when(fixture.factory.open(any())).thenAnswer(invocation -> {+ if (connections.getAndIncrement() == 0) {+ opened.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ }+ return DriverManager.getConnection(fixture.jdbcUrl);+ });+ ObjectNode request = fixture.single("expiry-active", "INSERT INTO mutations VALUES (1)");+ request.remove("WaitTimeSeconds");+ String original = fixture.service.executeStatement(request, "us-east-1").path("Id").asText();+ assertTrue(opened.await(5, TimeUnit.SECONDS));+ String replacement;+ try {+ fixture.clock.advance(Duration.ofHours(8));+ replacement = fixture.service.executeStatement(request.deepCopy().put("WaitTimeSeconds", 5), "us-east-1")+ .path("Id").asText();+ assertNotEquals(original, replacement);+ } finally {+ release.countDown();+ }+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertEquals(2, fixture.count()));+ assertEquals(replacement, fixture.service.executeStatement(request, "us-east-1").path("Id").asText());+ verify(fixture.factory, times(2)).open(any());+ }+ }++ @Test+ void waitingRetryDoesNotBlockAnotherTokensAdmission() throws Exception {+ try (Fixture fixture = new Fixture(); ExecutorService callers = Executors.newFixedThreadPool(2)) {+ CountDownLatch opened = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ CountDownLatch retryLookup = new CountDownLatch(1);+ AtomicInteger connections = new AtomicInteger();+ when(fixture.factory.open(any())).thenAnswer(invocation -> {+ if (connections.getAndIncrement() == 0) {+ opened.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ }+ return DriverManager.getConnection(fixture.jdbcUrl);+ });+ ObjectNode request = fixture.single("waiting", "INSERT INTO mutations VALUES (1)");+ request.remove("WaitTimeSeconds");+ String first = fixture.service.executeStatement(request, "us-east-1").path("Id").asText();+ assertTrue(opened.await(5, TimeUnit.SECONDS));+ doAnswer(invocation -> {+ retryLookup.countDown();+ return invocation.callRealMethod();+ }).when(fixture.receipts).find(any());+ Future<ObjectNode> retry = callers.submit(() -> fixture.service.executeStatement(+ request.deepCopy().put("WaitTimeSeconds", 5), "us-east-1"));+ try {+ assertTrue(retryLookup.await(5, TimeUnit.SECONDS));+ Future<ObjectNode> independent = callers.submit(() -> fixture.service.executeStatement(+ fixture.single("independent", "SELECT 1"), "us-east-1"));+ assertNotEquals(first, independent.get(2, TimeUnit.SECONDS).path("Id").asText());+ } finally {+ release.countDown();+ }+ assertEquals(first, retry.get(5, TimeUnit.SECONDS).path("Id").asText());+ }+ }++ private RedshiftDataIdempotencyStore persistentReceipts(Path file, Clock clock) {+ PersistentStorage<String, RedshiftDataIdempotencyStore.Receipt> delegate = new PersistentStorage<>(file,+ new TypeReference<Map<String, RedshiftDataIdempotencyStore.Receipt>>() { });+ delegate.load();+ return new RedshiftDataIdempotencyStore(new AccountAwareStorageBackend<>(delegate, null, "000000000000"), clock);+ }++ private RedshiftDataStatementStore persistentStatements(Path file, RedshiftDataIdempotencyStore receipts) {+ PersistentStorage<String, RedshiftDataStatementStore.StoredStatement> delegate = new PersistentStorage<>(file,+ new TypeReference<Map<String, RedshiftDataStatementStore.StoredStatement>>() { });+ delegate.load();+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.<RedshiftDataStatementStore.StoredStatement>create(eq("redshift-data"), eq("statements.json"), any()))+ .thenReturn(new AccountAwareStorageBackend<>(delegate, null, "000000000000"));+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().redshiftData().resultTtlHours()).thenReturn(24);+ return new RedshiftDataStatementStore(factory, config, receipts);+ }++ private static final class Fixture implements AutoCloseable {+ private final ObjectMapper mapper = new ObjectMapper();+ private final String jdbcUrl = "jdbc:h2:mem:" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1";+ private final RedshiftDataConnectionFactory factory = mock(RedshiftDataConnectionFactory.class);+ private final RedshiftDataEventPublisher publisher = mock(RedshiftDataEventPublisher.class);+ private final RedshiftDataExecutor executor = new RedshiftDataExecutor(2, 16, 1);+ private final MutableClock clock = new MutableClock();+ private final RedshiftDataStatementStore statements = spy(new RedshiftDataStatementStore(1, clock));+ private final RedshiftDataIdempotencyStore receipts = spy(new RedshiftDataIdempotencyStore(+ AccountAwareStorageBackend.inMemory("000000000000"), clock));+ private final RedshiftDataService service;+ private final RedshiftDataResourceResolver resolver = mock(RedshiftDataResourceResolver.class);++ private Fixture() throws Exception {+ Driver.load();+ try (Connection connection = DriverManager.getConnection(jdbcUrl); Statement statement = connection.createStatement()) {+ statement.execute("CREATE TABLE mutations (id INT)");+ }+ when(resolver.resolve(any(), any())).thenReturn(new RedshiftDataResourceResolver.DatabaseTarget(+ "arn:aws:redshift:us-east-1:000000000000:cluster:wh", "localhost", 5439, "dev", "admin", "password"));+ when(factory.open(any())).thenAnswer(invocation -> DriverManager.getConnection(jdbcUrl));+ service = service(statements, receipts);+ }++ private RedshiftDataService service(RedshiftDataStatementStore statements, RedshiftDataIdempotencyStore receipts) {+ return new RedshiftDataService(resolver, factory, statements, mapper, null, executor, publisher,+ null, null, receipts);+ }++ private ObjectNode single(String token, String sql) {+ return mapper.createObjectNode().put("ClientToken", token).put("Sql", sql).put("WaitTimeSeconds", 5);+ }++ private int count() throws Exception {+ try (Connection connection = DriverManager.getConnection(jdbcUrl);+ Statement statement = connection.createStatement();+ ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM mutations")) {+ assertTrue(rows.next());+ return rows.getInt(1);+ }+ }++ @Override+ public void close() {+ executor.close();+ }+ }++ private static final class MutableClock extends Clock {+ private Instant now = Instant.now();++ private void advance(Duration duration) {+ now = now.plus(duration);+ }++ @Override+ public ZoneId getZone() {+ return ZoneOffset.UTC;+ }++ @Override+ public Clock withZone(ZoneId zone) {+ return Clock.fixed(now, zone);+ }++ @Override+ public Instant instant() {+ return now;+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStoreTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStoreTest.javanew file mode 100644index 0000000000..bc9a6f3e16--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStoreTest.java@@ -0,0 +1,206 @@+package io.github.hectorvent.floci.services.redshiftdata;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.HybridStorage;+import io.github.hectorvent.floci.core.storage.PersistentStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.WalStorage;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.nio.file.Path;+import java.time.Clock;+import java.time.Duration;+import java.time.Instant;+import java.time.ZoneOffset;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.*;++class RedshiftDataIdempotencyStoreTest {++ private static final Instant START = Instant.parse("2026-10-23T00:00:00Z");+ private final ObjectMapper mapper = new ObjectMapper();++ @Test+ void expirySweepDoesNotRewriteTheFileForEveryReceipt(@TempDir Path directory) {+ PersistentStorage<String, RedshiftDataIdempotencyStore.Receipt> delegate = spy(new PersistentStorage<>(+ directory.resolve("idempotency.json"), new TypeReference<Map<String, RedshiftDataIdempotencyStore.Receipt>>() { }));+ AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> backend =+ new AccountAwareStorageBackend<>(delegate, null, "000000000000");+ RedshiftDataIdempotencyStore initial = store(backend, START);+ for (int i = 0; i < 25; i++) {+ initial.put(scope(i % 2 == 0 ? "000000000000" : "111111111111", "us-east-1", "ExecuteStatement", "token-" + i),+ mapper.createObjectNode().put("Id", "statement-" + i));+ }+ clearInvocations(delegate);+ store(backend, START.plus(Duration.ofHours(8))).sweep();+ assertTrue(backend.scanAllAccountEntries(key -> true).isEmpty());+ verify(delegate, never()).delete(any());+ verify(delegate, never()).flush();+ verify(delegate, times(1)).deleteAllAtomically(any());+ }++ @Test+ void sweepSnapshotDoesNotBlockNewReceiptsOrDeleteAReplacement() throws Exception {+ AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> backend =+ spy(AccountAwareStorageBackend.<RedshiftDataIdempotencyStore.Receipt>inMemory("000000000000"));+ RedshiftDataIdempotencyStore.Scope scope = scope("000000000000", "us-east-1", "ExecuteStatement", "token");+ store(backend, START).put(scope, mapper.createObjectNode().put("Id", "expired"));+ RedshiftDataIdempotencyStore later = store(backend, START.plus(Duration.ofHours(8)));+ CountDownLatch scanned = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ Object entries = invocation.callRealMethod();+ scanned.countDown();+ assertTrue(release.await(5, TimeUnit.SECONDS));+ return entries;+ }).when(backend).scanAllAccountEntries(any());+ try (ExecutorService workers = Executors.newFixedThreadPool(2)) {+ Future<?> sweep = workers.submit(later::sweep);+ try {+ assertTrue(scanned.await(5, TimeUnit.SECONDS));+ Future<?> replacement = workers.submit(() -> later.put(scope, mapper.createObjectNode().put("Id", "replacement")));+ replacement.get(2, TimeUnit.SECONDS);+ } finally {+ release.countDown();+ }+ sweep.get(5, TimeUnit.SECONDS);+ assertEquals("replacement", later.find(scope).response.path("Id").asText());+ }+ }++ @Test+ void deadlineDoesNotSlideAndExpiresAtEightHours() {+ AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> backend =+ AccountAwareStorageBackend.inMemory("000000000000");+ RedshiftDataIdempotencyStore initial = store(backend, START);+ RedshiftDataIdempotencyStore.Scope scope = scope("000000000000", "us-east-1", "ExecuteStatement", "token");+ initial.put(scope, mapper.createObjectNode().put("Id", "original"));+ RedshiftDataIdempotencyStore later = store(backend, START.plus(Duration.ofHours(8)).minusNanos(1));+ assertEquals("original", later.find(scope).response.path("Id").asText());+ assertEquals(START.plus(Duration.ofHours(8)), later.find(scope).expiresAt);+ assertNull(store(backend, START.plus(Duration.ofHours(8))).find(scope));+ }++ @Test+ void scopesCannotCollideAndReceiptsAreIndependentSnapshots() {+ RedshiftDataIdempotencyStore store = store(AccountAwareStorageBackend.inMemory("000000000000"), START);+ RedshiftDataIdempotencyStore.Scope original = scope("000000000000", "us-east-1", "ExecuteStatement", "x:y/ z");+ ObjectNode response = mapper.createObjectNode().put("Id", "original");+ store.put(original, response);+ response.put("Id", "mutated");+ assertEquals("original", store.find(original).response.path("Id").asText());+ store.find(original).response.put("Id", "mutated-read");+ assertEquals("original", store.find(original).response.path("Id").asText());+ assertNull(store.find(scope("111111111111", "us-east-1", "ExecuteStatement", "x:y/ z")));+ assertNull(store.find(scope("000000000000", "eu-west-1", "ExecuteStatement", "x:y/ z")));+ assertNull(store.find(scope("000000000000", "us-east-1", "BatchExecuteStatement", "x:y/ z")));+ assertNull(store.find(scope("000000000000", "us-east-1", "ExecuteStatement", "X:y/ z")));+ }++ @Test+ void staleRemovalCannotDeleteAReplacementReceipt() {+ RedshiftDataIdempotencyStore store = store(AccountAwareStorageBackend.inMemory("000000000000"), START);+ RedshiftDataIdempotencyStore.Scope scope = scope("000000000000", "us-east-1", "ExecuteStatement", "token");+ store.put(scope, mapper.createObjectNode().put("Id", "new"));+ store.removeIfStatement(scope, "old");+ assertNotNull(store.find(scope));+ store.removeIfStatement(scope, "new");+ assertNull(store.find(scope));+ }++ @Test+ void sweepAndClearCoverEveryAccount() {+ AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> backend =+ AccountAwareStorageBackend.inMemory("000000000000");+ RedshiftDataIdempotencyStore store = store(backend, START);+ store.put(scope("000000000000", "us-east-1", "ExecuteStatement", "a"), mapper.createObjectNode().put("Id", "a"));+ store.put(scope("111111111111", "us-east-1", "ExecuteStatement", "a"), mapper.createObjectNode().put("Id", "b"));+ store(backend, START.plus(Duration.ofHours(8))).sweep();+ assertTrue(backend.scanAllAccountEntries(key -> true).isEmpty());+ store.put(scope("111111111111", "us-east-1", "ExecuteStatement", "a"), mapper.createObjectNode().put("Id", "c"));+ store.clear();+ assertTrue(backend.scanAllAccountEntries(key -> true).isEmpty());+ }++ @Test+ void receiptsSurvivePersistentBackendReopen(@TempDir Path directory) {+ Path file = directory.resolve("idempotency.json");+ RedshiftDataIdempotencyStore.Scope scope = scope("111111111111", "cn-north-1", "ExecuteStatement", "token");+ RedshiftDataIdempotencyStore original = store(persistent(file), START);+ assertNull(original.find(scope));+ original.put(scope, mapper.createObjectNode().put("Id", "persisted").put("CreatedAt", 123.25));+ RedshiftDataIdempotencyStore reopened = store(persistent(file), START.plusSeconds(60));+ assertEquals("persisted", reopened.find(scope).response.path("Id").asText());+ assertEquals(123.25, reopened.find(scope).response.path("CreatedAt").asDouble());+ assertEquals(START.plus(Duration.ofHours(8)), reopened.find(scope).expiresAt);+ }++ @ParameterizedTest+ @ValueSource(strings = {"hybrid", "wal"})+ void receiptsSurviveWriteBehindBackendRestart(String mode, @TempDir Path directory) {+ RedshiftDataIdempotencyStore.Scope scope = scope("111111111111", "us-east-1", "ExecuteStatement", "token");+ StorageBackend<String, RedshiftDataIdempotencyStore.Receipt> initial = writeBehind(mode, directory);+ initial.load();+ try {+ store(new AccountAwareStorageBackend<>(initial, null, "000000000000"), START)+ .put(scope, mapper.createObjectNode().put("Id", "persisted"));+ } finally {+ shutdown(initial);+ }+ StorageBackend<String, RedshiftDataIdempotencyStore.Receipt> reopened = writeBehind(mode, directory);+ reopened.load();+ try {+ assertEquals("persisted", store(new AccountAwareStorageBackend<>(reopened, null, "000000000000"), START)+ .find(scope).response.path("Id").asText());+ } finally {+ shutdown(reopened);+ }+ }++ private StorageBackend<String, RedshiftDataIdempotencyStore.Receipt> writeBehind(String mode, Path directory) {+ TypeReference<Map<String, RedshiftDataIdempotencyStore.Receipt>> type = new TypeReference<>() { };+ return switch (mode) {+ case "hybrid" -> new HybridStorage<>(directory.resolve("idempotency.json"), type, 60000);+ case "wal" -> new WalStorage<>(directory.resolve("idempotency.json"), directory.resolve("idempotency.wal"), type, 60000);+ default -> throw new IllegalArgumentException(mode);+ };+ }++ private void shutdown(StorageBackend<String, RedshiftDataIdempotencyStore.Receipt> backend) {+ if (backend instanceof HybridStorage<String, RedshiftDataIdempotencyStore.Receipt> hybrid) {+ hybrid.shutdown();+ } else if (backend instanceof WalStorage<String, RedshiftDataIdempotencyStore.Receipt> wal) {+ wal.shutdown();+ }+ }++ private AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> persistent(Path file) {+ PersistentStorage<String, RedshiftDataIdempotencyStore.Receipt> delegate = new PersistentStorage<>(file,+ new TypeReference<Map<String, RedshiftDataIdempotencyStore.Receipt>>() { });+ delegate.load();+ return new AccountAwareStorageBackend<>(delegate, null, "000000000000");+ }++ private RedshiftDataIdempotencyStore store(AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> backend,+ Instant now) {+ return new RedshiftDataIdempotencyStore(backend, Clock.fixed(now, ZoneOffset.UTC));+ }++ private RedshiftDataIdempotencyStore.Scope scope(String account, String region, String operation, String token) {+ return new RedshiftDataIdempotencyStore.Scope(account, region, operation, token);+ }+}