Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(redshift-data): support client token idempotency (#6343)

Vũ Thế Huy38c641ec7e37
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java
index 6e35a3d2e1..52635353c1 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java
@@ -13,6 +13,7 @@ import software.amazon.awssdk.services.redshift.RedshiftClient;
import software.amazon.awssdk.services.redshift.model.CreateClusterRequest;
import software.amazon.awssdk.services.redshift.model.DeleteClusterRequest;
import software.amazon.awssdk.services.redshiftdata.RedshiftDataClient;
+import software.amazon.awssdk.services.redshiftdata.model.BatchExecuteStatementRequest;
import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementRequest;
import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementResponse;
import software.amazon.awssdk.services.redshiftdata.model.ExecuteStatementRequest;
@@ -74,6 +75,37 @@ class RedshiftDataOperationsTest {
}
}
+ @Test
+ @DisplayName("explicit client token retries an INSERT without inserting twice")
+ void executeStatementClientTokenRetry() throws Exception {
+ run("CREATE TABLE compat_idempotent_single (id int)");
+ ExecuteStatementRequest request = ExecuteStatementRequest.builder()
+ .clusterIdentifier(clusterId).database(DATABASE).dbUser(USERNAME)
+ .clientToken(TestFixtures.uniqueName("single-token"))
+ .sql("INSERT INTO compat_idempotent_single VALUES (1)").build();
+ String first = data.executeStatement(request).id();
+ awaitFinished(first);
+ assertThat(data.executeStatement(request).id()).isEqualTo(first);
+ String count = run("SELECT COUNT(*) FROM compat_idempotent_single");
+ assertThat(data.getStatementResult(r -> r.id(count)).records().get(0).get(0).longValue()).isEqualTo(1L);
+ }
+
+ @Test
+ @DisplayName("explicit client token retries a batch without repeating its mutations")
+ void batchExecuteStatementClientTokenRetry() throws Exception {
+ run("CREATE TABLE compat_idempotent_batch (id int)");
+ BatchExecuteStatementRequest request = BatchExecuteStatementRequest.builder()
+ .clusterIdentifier(clusterId).database(DATABASE).dbUser(USERNAME)
+ .clientToken(TestFixtures.uniqueName("batch-token"))
+ .sqls("INSERT INTO compat_idempotent_batch VALUES (1)",
+ "INSERT INTO compat_idempotent_batch VALUES (2)").build();
+ String first = data.batchExecuteStatement(request).id();
+ awaitFinished(first);
+ assertThat(data.batchExecuteStatement(request).id()).isEqualTo(first);
+ String count = run("SELECT COUNT(*) FROM compat_idempotent_batch");
+ assertThat(data.getStatementResult(r -> r.id(count)).records().get(0).get(0).longValue()).isEqualTo(2L);
+ }
+
@Test
@DisplayName("execute, describe, and get-statement-result round trip")
void executeDescribeGetResult() throws Exception {
diff --git a/docs/services/redshift-data.md b/docs/services/redshift-data.md
index f57935cc94..0ff6337ba9 100644
--- a/docs/services/redshift-data.md
+++ b/docs/services/redshift-data.md
@@ -58,6 +58,48 @@ A statement run against a workgroup reports `WorkgroupName` and omits `ClusterId
- **`ExecuteSql` and `BatchExecuteSql`** (the deprecated pre-2020 operations) return `ValidationException`.
- **Type mapping.** JDBC `BOOLEAN` and `BIT` map to `booleanValue`; integer types to `longValue`; floating-point types to `doubleValue`; `NUMERIC` and `DECIMAL` to `stringValue` (as AWS does); binary types to `blobValue`; everything else, including dates, timestamps, and uuid, to `stringValue`. A SQL `NULL` maps to `isNull`. A result column of type `line`, `json`, or `jsonb` fails the statement with the Redshift error text.
+## Client token retries
+
+`ExecuteStatement` and `BatchExecuteStatement` accept an optional `ClientToken`, a case-sensitive
+string of 1 to 64 characters. Repeating an admitted request with the same token returns the
+original statement `Id` and `CreatedAt` without running SQL or publishing its terminal event
+again. This applies while the statement is queued or running and after it finishes, fails or is
+cancelled. Omit the token, or use a new one, to submit independent work. AWS SDKs normally generate
+the token automatically and keep it for their retries.
+
+Tokens expire 8 hours after first admission. Retries do not extend this deadline. Validation,
+reported storage errors and queue-admission failures do not consume the token. An SQL execution failure after
+admission does consume it. `WaitTimeSeconds` on a retry can wait for the original execution without
+submitting another one.
+
+Request validation and database-target resolution run before admission coordination. Only requests
+sharing an account, region, execute operation and token share an admission lock; tokenless requests
+use no token lock. The executor reserves queue capacity before writing admission state, so a full
+queue cannot create a receipt that reappears after restart. Workers run SQL only after admission
+writes succeed. Reset still signals cancellation if clearing either storage backend reports an
+error. Expired receipts are removed in one atomic batch per sweep, with concurrently replaced
+receipts retained.
+
+Receipts use the configured storage mode. Persistent, hybrid and WAL modes retain them across a
+normal restart; memory mode loses them. Interrupted statements keep the existing recovery behavior:
+they become `FAILED`, and SQL is never replayed automatically. This does not provide a distributed
+transaction between PostgreSQL and Floci storage or an exactly-once guarantee after a host crash.
+Persistent and WAL modes use atomic admission writes that report write failures before enqueueing
+SQL. Hybrid mode retains its write-behind behavior: disk flush failures are logged while the live
+in-memory state remains available.
+The token deadline is independent of result retention: if a configured result TTL shorter than
+8 hours removes the statement, its receipt still returns the original admission response until
+the token expires, without restoring results.
+
+Floci scopes tokens by account, region and execute operation. It validates the request and resolves
+the database target before looking up a receipt, including on retries. Within that scope, reusing
+a token with changed SQL, parameters or event settings replays the original operation; it does not
+change that operation. The precise AWS behavior for changed payloads, different IAM principals,
+cross-operation reuse and responses after result eviction has not been verified against a live
+AWS cluster. These are explicit Floci policies. See the
+[AWS Data API guide](https://docs.aws.amazon.com/redshift/latest/mgmt/data-api.html) for the documented
+8-hour token window and SDK retry behavior.
+
## Statement status events
[Redshift Data API status events](https://docs.aws.amazon.com/redshift/latest/mgmt/data-api-monitoring-events.html)
diff --git a/pom.xml b/pom.xml
index 2ad1956285..ccca191e64 100644
--- a/pom.xml
+++ b/pom.xml
@@ -417,6 +417,12 @@
<version>${aws.sdk.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>redshiftdata</artifactId>
+ <version>${aws.sdk.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>url-connection-client</artifactId>
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
index e9c0cd54f7..4f877ccddf 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
@@ -438,6 +438,17 @@ public class AccountAwareStorageBackend<V> implements StorageBackend<String, V>
delegate.delete(accountId + "/" + key);
}
+ @Override
+ public void deleteAllAtomically(Set<String> keys) {
+ deleteAllAtomicallyForAccounts(Map.of(prefix(), keys));
+ }
+
+ public void deleteAllAtomicallyForAccounts(Map<String, Set<String>> keysByAccount) {
+ Set<String> rawKeys = new LinkedHashSet<>();
+ keysByAccount.forEach((account, keys) -> keys.forEach(key -> rawKeys.add(account + "/" + key)));
+ delegate.deleteAllAtomically(rawKeys);
+ }
+
// --- Entry writes, for map-valued stores whose backend persists one entry at a time ---
/** Whether the backend persists one map entry at a time; the entry methods require it. */
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java
index ff1675c686..5245097e8c 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java
@@ -72,6 +72,17 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {
dirty.set(true);
}
+ @Override
+ public synchronized void deleteAllAtomically(Set<K> keys) {
+ if (keys.isEmpty()) {
+ return;
+ }
+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);
+ candidate.keySet().removeAll(Set.copyOf(keys));
+ store = candidate;
+ dirty.set(true);
+ }
+
@Override
public List<V> scan(Predicate<K> keyFilter) {
return store.entrySet().stream()
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java
index 630e9ee6d4..d645202b74 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java
@@ -38,6 +38,16 @@ public class InMemoryStorage<K, V> implements StorageBackend<K, V> {
store.remove(key);
}
+ @Override
+ public synchronized void deleteAllAtomically(Set<K> keys) {
+ if (keys.isEmpty()) {
+ return;
+ }
+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);
+ candidate.keySet().removeAll(Set.copyOf(keys));
+ store = candidate;
+ }
+
@Override
public List<V> scan(Predicate<K> keyFilter) {
List<V> result = new ArrayList<>();
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java
index 8df755766e..a23160a19d 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java
@@ -85,6 +85,21 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {
persistToDisk();
}
+ @Override
+ public synchronized void deleteAllAtomically(Set<K> keys) {
+ if (keys.isEmpty()) {
+ return;
+ }
+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);
+ candidate.keySet().removeAll(Set.copyOf(keys));
+ try {
+ StorageFiles.writeAtomically(filePath, tempFile -> objectMapper.writeValue(tempFile.toFile(), candidate));
+ } catch (IOException exception) {
+ throw new UncheckedIOException("Failed to commit atomic deletions to " + filePath, exception);
+ }
+ store = candidate;
+ }
+
@Override
public List<V> scan(Predicate<K> keyFilter) {
return store.entrySet().stream()
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java
index c7ba2ab5d8..02c9ceb9fb 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java
@@ -39,6 +39,11 @@ public interface StorageBackend<K, V> {
void delete(K key);
+ /** Remove keys as one storage commit, with the same durability guarantees as atomic puts. */
+ default void deleteAllAtomically(Set<K> keys) {
+ throw new UnsupportedOperationException("Atomic deletions are not supported by this backend");
+ }
+
/**
* Return a new mutable list of values whose keys pass the filter. Callers may sort,
* filter, or otherwise mutate the returned list without affecting the underlying store.
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java
index 54b295452b..abd1ec7e50 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java
@@ -43,6 +43,7 @@ import java.util.stream.Collectors;
* PUT: [0x01] [4-byte key length] [key bytes] [4-byte value length] [value bytes]
* DELETE: [0x02] [4-byte key length] [key bytes]
* BATCH: [0x03] [4-byte map length] [CBOR map bytes]
+ * DELETE_BATCH: [0x04] [4-byte array length] [CBOR key array bytes]
*
* Key and value bytes are serialized via Jackson CBOR (compact binary format).
* Snapshot files use indented JSON for debuggability.
@@ -54,6 +55,7 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {
static final byte OP_PUT = 0x01;
static final byte OP_DELETE = 0x02;
static final byte OP_BATCH = 0x03;
+ static final byte OP_DELETE_BATCH = 0x04;
private volatile ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();
private final Path snapshotPath;
@@ -130,6 +132,25 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {
} catch (IOException exception) {
throw new UncheckedIOException("Failed to encode atomic WAL batch", exception);
}
+ commitBatch(OP_BATCH, payload, replacements, Set.of());
+ }
+
+ @Override
+ public void deleteAllAtomically(Set<K> keys) {
+ Set<K> deletions = Set.copyOf(keys);
+ if (deletions.isEmpty()) {
+ return;
+ }
+ byte[] payload;
+ try {
+ payload = walMapper.writeValueAsBytes(deletions);
+ } catch (IOException exception) {
+ throw new UncheckedIOException("Failed to encode atomic WAL deletions", exception);
+ }
+ commitBatch(OP_DELETE_BATCH, payload, Map.of(), deletions);
+ }
+
+ private void commitBatch(byte operation, byte[] payload, Map<K, V> replacements, Set<K> deletions) {
compactionLock.writeLock().lock();
try {
if (walWriter == null) {
@@ -137,13 +158,14 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {
}
ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);
candidate.putAll(replacements);
+ candidate.keySet().removeAll(deletions);
walWriter.flush();
FileChannel channel = FileChannel.open(walPath, StandardOpenOption.WRITE);
try {
long offset = channel.size();
channel.position(offset);
ByteBuffer record = ByteBuffer.allocate(1 + Integer.BYTES + payload.length);
- record.put(OP_BATCH).putInt(payload.length).put(payload).flip();
+ record.put(operation).putInt(payload.length).put(payload).flip();
try {
while (record.hasRemaining()) {
channel.write(record);
@@ -417,6 +439,14 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {
break;
}
store.putAll(batch);
+ } else if (op == OP_DELETE_BATCH) {
+ List<K> keys = walMapper.readValue(keyBytes, walMapper.getTypeFactory().constructCollectionType(
+ List.class, walMapper.constructType(typeReference.getType()).getKeyType()));
+ if (keys == null || keys.stream().anyMatch(key -> key == null)) {
+ LOG.warnv("Invalid WAL deletion batch at offset {0} in {1}, stopping replay", consumed, walPath);
+ break;
+ }
+ keys.forEach(store::remove);
} else if (op == OP_PUT) {
byte[] valueBytes = readFrame(in, fileSize - consumed - recordLength);
if (valueBytes == null) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutor.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutor.java
index c3fbcd49b2..f323e94aef 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutor.java
@@ -14,12 +14,15 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
+import java.util.function.Supplier;
@ApplicationScoped
class RedshiftDataExecutor implements AutoCloseable {
@@ -65,8 +68,9 @@ class RedshiftDataExecutor implements AutoCloseable {
}
/**
- * Admits a statement. {@code admit} runs under the reset read lock, so a reset cannot slip
- * between persisting the statement and registering its execution. {@code onFailure} runs when
+ * Reserves queue capacity before {@code admit} writes any state. The worker waits for admission
+ * to succeed. {@code admit} runs under the reset read lock, so reset cannot split admission.
+ * {@code onFailure} runs when
* the worker dies with an unexpected exception, so the statement can still reach a terminal state.
*/
RedshiftDataExecution submit(String account, String region, String id, Runnable admit,
@@ -79,7 +83,25 @@ class RedshiftDataExecutor implements AutoCloseable {
}
String key = account + ":" + id;
RedshiftDataExecution execution = new RedshiftDataExecution(generation);
+ CountDownLatch ready = new CountDownLatch(1);
+ AtomicBoolean admitted = new AtomicBoolean();
+ AtomicBoolean claimed = new AtomicBoolean();
Runnable task = () -> {
+ if (closed && ready.getCount() != 0) {
+ return;
+ }
+ try {
+ ready.await();
+ } catch (InterruptedException expected) {
+ // The admitting thread completes a pending reservation after shutdown interrupts its worker.
+ Thread.currentThread().interrupt();
+ if (ready.getCount() != 0) {
+ return;
+ }
+ }
+ if (!admitted.get() || !claimed.compareAndSet(false, true)) {
+ return;
+ }
try {
RequestScopes.runAs(account, region, () -> work.accept(execution));
} catch (RuntimeException failure) {
@@ -93,7 +115,6 @@ class RedshiftDataExecutor implements AutoCloseable {
}
};
execution.task(task);
- admit.run();
synchronized (this) {
executions.put(key, execution);
}
@@ -105,12 +126,45 @@ class RedshiftDataExecutor implements AutoCloseable {
}
throw new AwsException("ActiveStatementsExceededException", "Too many active statements", 400);
}
+ try {
+ admit.run();
+ admitted.set(true);
+ } catch (RuntimeException | Error failure) {
+ workers.remove(task);
+ synchronized (this) {
+ executions.remove(key, execution);
+ }
+ execution.completed();
+ throw failure;
+ } finally {
+ boolean finishAfterShutdown;
+ synchronized (this) {
+ finishAfterShutdown = closed && admitted.get();
+ if (finishAfterShutdown) {
+ execution.requestCancellation();
+ }
+ ready.countDown();
+ }
+ if (finishAfterShutdown) {
+ Thread.startVirtualThread(task);
+ }
+ }
return execution;
} finally {
resetLock.readLock().unlock();
}
}
+ /** Keeps receipt lookup, admission and rejected-admission cleanup in one reset generation. */
+ <T> T admitting(Supplier<T> admission) {
+ resetLock.readLock().lock();
+ try {
+ return admission.get();
+ } finally {
+ resetLock.readLock().unlock();
+ }
+ }
+
private static void recover(RedshiftDataExecution execution, RuntimeException failure,
BiConsumer<RedshiftDataExecution, RuntimeException> onFailure,
String id, String account, String region) {
@@ -121,6 +175,10 @@ class RedshiftDataExecutor implements AutoCloseable {
}
}
+ synchronized RedshiftDataExecution execution(String account, String id) {
+ return executions.get(account + ":" + id);
+ }
+
boolean cancel(String account, String id) {
RedshiftDataExecution execution;
synchronized (this) {
@@ -173,12 +231,12 @@ class RedshiftDataExecutor implements AutoCloseable {
}
void clear(Runnable clearStorage) {
- List<RedshiftDataExecution> active;
+ List<RedshiftDataExecution> active = new ArrayList<>();
resetLock.writeLock().lock();
try {
generation++;
synchronized (this) {
- active = new ArrayList<>(executions.values());
+ active.addAll(executions.values());
executions.clear();
}
for (RedshiftDataExecution execution : active) {
@@ -187,13 +245,13 @@ class RedshiftDataExecutor implements AutoCloseable {
clearStorage.run();
} finally {
resetLock.writeLock().unlock();
- }
- for (RedshiftDataExecution execution : active) {
- if (execution.requestCancellation()) {
- Thread.startVirtualThread(execution::signalCancellation);
- }
- if (workers.remove(execution.task())) {
- execution.completed();
+ for (RedshiftDataExecution execution : active) {
+ if (execution.requestCancellation()) {
+ Thread.startVirtualThread(execution::signalCancellation);
+ }
+ if (workers.remove(execution.task())) {
+ execution.completed();
+ }
}
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStore.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStore.java
new file mode 100644
index 0000000000..ec8a92a1c6
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStore.java
@@ -0,0 +1,149 @@
+package io.github.hectorvent.floci.services.redshiftdata;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Base64;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+@ApplicationScoped
+class RedshiftDataIdempotencyStore {
+ private static final Duration TOKEN_TTL = Duration.ofHours(8);
+
+ private final AccountAwareStorageBackend<Receipt> receipts;
+ private final Clock clock;
+ private final ReentrantReadWriteLock mutations = new ReentrantReadWriteLock();
+
+ @Inject
+ RedshiftDataIdempotencyStore(StorageFactory factory) {
+ this(factory.create("redshift-data", "idempotency.json", new TypeReference<Map<String, Receipt>>() { }),
+ Clock.systemUTC());
+ }
+
+ RedshiftDataIdempotencyStore(AccountAwareStorageBackend<Receipt> receipts, Clock clock) {
+ this.receipts = receipts;
+ this.clock = clock;
+ }
+
+ static String clientToken(JsonNode request) {
+ JsonNode token = request.get("ClientToken");
+ if (token == null) {
+ return null;
+ }
+ if (!token.isTextual() || token.textValue().isEmpty() || token.textValue().length() > 64) {
+ throw new AwsException("ValidationException", "ClientToken must be a string between 1 and 64 characters", 400);
+ }
+ return token.textValue();
+ }
+
+ Receipt find(Scope scope) {
+ Receipt receipt = receipts.getForAccount(scope.accountId(), scope.key()).orElse(null);
+ if (receipt != null && !clock.instant().isBefore(receipt.expiresAt)) {
+ return null;
+ }
+ return receipt == null ? null : receipt.snapshot();
+ }
+
+ void put(Scope scope, ObjectNode response) {
+ Receipt receipt = new Receipt();
+ receipt.acceptedAt = clock.instant();
+ receipt.expiresAt = receipt.acceptedAt.plus(TOKEN_TTL);
+ receipt.response = response.deepCopy();
+ mutations.readLock().lock();
+ try {
+ receipts.putAllAtomicallyForAccount(scope.accountId(), Map.of(scope.key(), receipt));
+ } finally {
+ mutations.readLock().unlock();
+ }
+ }
+
+ void removeIfStatement(Scope scope, String statementId) {
+ mutations.writeLock().lock();
+ try {
+ Receipt receipt = receipts.getForAccount(scope.accountId(), scope.key()).orElse(null);
+ if (receipt != null && statementId.equals(receipt.response.path("Id").asText())) {
+ receipts.deleteAllAtomicallyForAccounts(Map.of(scope.accountId(), Set.of(scope.key())));
+ }
+ } finally {
+ mutations.writeLock().unlock();
+ }
+ }
+
+ void clear() {
+ mutations.writeLock().lock();
+ try {
+ Map<String, Set<String>> keys = new LinkedHashMap<>();
+ for (AccountAwareStorageBackend.AccountEntry<Receipt> entry : receipts.scanAllAccountEntries(key -> true)) {
+ keys.computeIfAbsent(entry.accountId(), ignored -> new LinkedHashSet<>()).add(entry.key());
+ }
+ receipts.deleteAllAtomicallyForAccounts(keys);
+ } finally {
+ mutations.writeLock().unlock();
+ }
+ }
+
+ void sweep() {
+ Instant now = clock.instant();
+ List<AccountAwareStorageBackend.AccountEntry<Receipt>> expired = receipts.scanAllAccountEntries(key -> true).stream()
+ .filter(entry -> !now.isBefore(entry.value().expiresAt)).toList();
+ if (expired.isEmpty()) {
+ return;
+ }
+ mutations.writeLock().lock();
+ try {
+ Map<String, Set<String>> keys = new LinkedHashMap<>();
+ for (AccountAwareStorageBackend.AccountEntry<Receipt> entry : expired) {
+ Receipt current = receipts.getForAccount(entry.accountId(), entry.key()).orElse(null);
+ if (current == entry.value()) {
+ keys.computeIfAbsent(entry.accountId(), ignored -> new LinkedHashSet<>()).add(entry.key());
+ }
+ }
+ receipts.deleteAllAtomicallyForAccounts(keys);
+ } finally {
+ mutations.writeLock().unlock();
+ }
+ }
+
+ record Scope(String accountId, String region, String operation, String token) {
+ String key() {
+ return encode(region) + "." + encode(operation) + "." + encode(token);
+ }
+
+ private static String encode(String value) {
+ return Base64.getUrlEncoder().withoutPadding().encodeToString(value.getBytes(StandardCharsets.UTF_8));
+ }
+ }
+
+ @RegisterForReflection
+ @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
+ static final class Receipt {
+ Instant acceptedAt;
+ Instant expiresAt;
+ ObjectNode response;
+
+ Receipt snapshot() {
+ Receipt copy = new Receipt();
+ copy.acceptedAt = acceptedAt;
+ copy.expiresAt = expiresAt;
+ copy.response = response.deepCopy();
+ return copy;
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java
index 3c0a8333fe..9919789763 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java
@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RequestContext;
import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.common.SqlParameterParser.ParsedSql;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.redshift.spectrum.BackendSql;
import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumQueryPreparation;
@@ -24,6 +25,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
+import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
@@ -32,6 +34,8 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
import java.util.regex.Pattern;
@@ -54,18 +58,22 @@ public class RedshiftDataService implements Resettable {
private final RedshiftDataEventPublisher publisher;
private final RequestContext requestContext;
private final IamService iamService;
+ private final RedshiftDataIdempotencyStore idempotency;
+ private final ConcurrentHashMap<RedshiftDataIdempotencyStore.Scope, AdmissionLock> admissionLocks = new ConcurrentHashMap<>();
RedshiftDataService(RedshiftDataResourceResolver resolver, RedshiftDataConnectionFactory connectionFactory,
RedshiftDataStatementStore store, ObjectMapper objectMapper, SpectrumQueryPreparation preparation,
RedshiftDataExecutor executor, RedshiftDataEventPublisher publisher) {
- this(resolver, connectionFactory, store, objectMapper, preparation, executor, publisher, null, null);
+ this(resolver, connectionFactory, store, objectMapper, preparation, executor, publisher, null,
+ null, new RedshiftDataIdempotencyStore(AccountAwareStorageBackend.inMemory("000000000000"),
+ Clock.systemUTC()));
}
@Inject
public RedshiftDataService(RedshiftDataResourceResolver resolver, RedshiftDataConnectionFactory connectionFactory,
RedshiftDataStatementStore store, ObjectMapper objectMapper, SpectrumQueryPreparation preparation,
RedshiftDataExecutor executor, RedshiftDataEventPublisher publisher,
- RequestContext requestContext, IamService iamService) {
+ RequestContext requestContext, IamService iamService, RedshiftDataIdempotencyStore idempotency) {
this.resolver = resolver;
this.connectionFactory = connectionFactory;
this.store = store;
@@ -75,11 +83,31 @@ public class RedshiftDataService implements Resettable {
this.publisher = publisher;
this.requestContext = requestContext;
this.iamService = iamService;
+ this.idempotency = idempotency;
}
@Override
public void clear() {
- executor.clear(store::clear);
+ executor.clear(() -> {
+ RuntimeException failure = null;
+ try {
+ store.clear();
+ } catch (RuntimeException storageFailure) {
+ failure = storageFailure;
+ }
+ try {
+ idempotency.clear();
+ } catch (RuntimeException receiptFailure) {
+ if (failure == null) {
+ failure = receiptFailure;
+ } else {
+ failure.addSuppressed(receiptFailure);
+ }
+ }
+ if (failure != null) {
+ throw failure;
+ }
+ });
}
// ── ExecuteStatement ────────────────────────────────────────────────────
@@ -89,11 +117,58 @@ public class RedshiftDataService implements Resettable {
}
public ObjectNode executeStatement(JsonNode request, String region, String principal) {
+ return execute(request, region, principal, false);
+ }
+
+ private ObjectNode execute(JsonNode request, String region, String principal, boolean batch) {
+ String token = RedshiftDataIdempotencyStore.clientToken(request);
int waitSeconds = waitSeconds(request);
- return submit(prepareStatement(request, region), waitSeconds, principal);
+ PendingStatement pending = batch ? prepareBatch(request, region) : prepareStatement(request, region);
+ RedshiftDataStatementStore.StoredStatement stored = pending.statement();
+ RedshiftDataIdempotencyStore.Scope scope = token == null ? null : new RedshiftDataIdempotencyStore.Scope(
+ stored.accountId, region, batch ? "BatchExecuteStatement" : "ExecuteStatement", token);
+ Admission admission = executor.admitting(() -> scope == null ? submit(pending, principal, null)
+ : admitScoped(pending, principal, scope));
+ if (waitSeconds > 0 && admission.execution() != null) {
+ admission.execution().await(Duration.ofSeconds(waitSeconds));
+ RedshiftDataStatementStore.StoredStatement current = store.getForAccount(admission.accountId(), admission.id());
+ if (current != null) {
+ return executeResponse(current);
+ }
+ }
+ return admission.response();
+ }
+
+ private record Admission(String accountId, String id, ObjectNode response, RedshiftDataExecution execution) { }
+
+ private static final class AdmissionLock {
+ private final ReentrantLock lock = new ReentrantLock();
+ private int users;
}
- private ObjectNode submit(PendingStatement pending, int waitSeconds, String principal) {
+ private Admission admitScoped(PendingStatement pending, String principal, RedshiftDataIdempotencyStore.Scope scope) {
+ AdmissionLock scoped = admissionLocks.compute(scope, (key, existing) -> {
+ AdmissionLock selected = existing == null ? new AdmissionLock() : existing;
+ selected.users++;
+ return selected;
+ });
+ scoped.lock.lock();
+ try {
+ RedshiftDataIdempotencyStore.Receipt receipt = idempotency.find(scope);
+ if (receipt == null) {
+ return submit(pending, principal, scope);
+ }
+ String id = receipt.response.path("Id").asText();
+ RedshiftDataStatementStore.StoredStatement current = store.getForAccount(scope.accountId(), id);
+ return new Admission(scope.accountId(), id, current == null ? receipt.response : executeResponse(current),
+ executor.execution(scope.accountId(), id));
+ } finally {
+ scoped.lock.unlock();
+ admissionLocks.compute(scope, (key, existing) -> --scoped.users == 0 ? null : scoped);
+ }
+ }
+
+ private Admission submit(PendingStatement pending, String principal, RedshiftDataIdempotencyStore.Scope scope) {
RedshiftDataStatementStore.StoredStatement stored = pending.statement();
if (principal != null) {
stored.principal = principal;
@@ -102,7 +177,14 @@ public class RedshiftDataService implements Resettable {
RedshiftDataExecution handle;
try {
handle = executor.submit(stored.accountId, stored.region, stored.id,
- () -> store.putForAccount(stored.accountId, stored),
+ () -> {
+ if (scope == null) {
+ store.putForAccount(stored.accountId, stored);
+ } else {
+ store.admitForAccount(stored.accountId, stored);
+ idempotency.put(scope, response);
+ }
+ },
execution -> {
try {
execution.checkCancellation();
@@ -116,18 +198,22 @@ public class RedshiftDataService implements Resettable {
complete(stored, execution);
},
(execution, failure) -> recoverStatement(stored, execution, failure));
- } catch (AwsException failure) {
- store.deleteForAccount(stored.accountId, stored.id);
- throw failure;
- }
- if (waitSeconds > 0) {
- handle.await(Duration.ofSeconds(waitSeconds));
- RedshiftDataStatementStore.StoredStatement current = store.getForAccount(stored.accountId, stored.id);
- if (current != null) {
- return executeResponse(current);
+ } catch (RuntimeException failure) {
+ try {
+ if (scope != null) {
+ idempotency.removeIfStatement(scope, stored.id);
+ }
+ } catch (RuntimeException cleanupFailure) {
+ failure.addSuppressed(cleanupFailure);
}
+ try {
+ store.deleteForAccount(stored.accountId, stored.id);
+ } catch (RuntimeException cleanupFailure) {
+ failure.addSuppressed(cleanupFailure);
+ }
+ throw failure;
}
- return response;
+ return new Admission(stored.accountId, stored.id, response, handle);
}
private void complete(RedshiftDataStatementStore.StoredStatement stored, RedshiftDataExecution execution) {
@@ -477,8 +563,7 @@ public class RedshiftDataService implements Resettable {
}
public ObjectNode batchExecuteStatement(JsonNode request, String region, String principal) {
- int waitSeconds = waitSeconds(request);
- return submit(prepareBatch(request, region), waitSeconds, principal);
+ return execute(request, region, principal, true);
}
private PendingStatement prepareBatch(JsonNode request, String region) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataStatementStore.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataStatementStore.java
index 407b5fffa1..4c0102b0e2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataStatementStore.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataStatementStore.java
@@ -42,22 +42,25 @@ class RedshiftDataStatementStore {
private final AccountAwareStorageBackend<StoredStatement> statements;
private final Duration ttl;
private final Clock clock;
+ private final RedshiftDataIdempotencyStore idempotency;
// Created by the CDI lifecycle in start(); the test constructor never starts it, so unit
// tests that build the store directly drive sweep() by hand and spawn no background thread.
private ScheduledExecutorService sweeper;
@Inject
- RedshiftDataStatementStore(StorageFactory storageFactory, EmulatorConfig config) {
+ RedshiftDataStatementStore(StorageFactory storageFactory, EmulatorConfig config, RedshiftDataIdempotencyStore idempotency) {
this.statements = storageFactory.create("redshift-data", "statements.json",
new TypeReference<Map<String, StoredStatement>>() {});
this.ttl = Duration.ofHours(Math.max(1, config.services().redshiftData().resultTtlHours()));
this.clock = Clock.systemUTC();
+ this.idempotency = idempotency;
}
RedshiftDataStatementStore(int resultTtlHours, Clock clock) {
this.statements = AccountAwareStorageBackend.inMemory("000000000000");
this.ttl = Duration.ofHours(Math.max(1, resultTtlHours));
this.clock = clock;
+ this.idempotency = null;
}
@PostConstruct
@@ -95,10 +98,18 @@ class RedshiftDataStatementStore {
statements.putForAccount(accountId, statement.id, statement.snapshot());
}
+ void admitForAccount(String accountId, StoredStatement statement) {
+ statements.putAllAtomicallyForAccount(accountId, Map.of(statement.id, statement.snapshot()));
+ }
+
void deleteForAccount(String accountId, String id) {
statements.deleteForAccount(accountId, id);
}
+ void flush() {
+ statements.flush();
+ }
+
Instant expiresAt(Instant createdAt) {
return createdAt.plus(ttl);
}
@@ -120,6 +131,9 @@ class RedshiftDataStatementStore {
}
void sweep() {
+ if (idempotency != null) {
+ idempotency.sweep();
+ }
Instant cutoff = Instant.now(clock).minus(ttl);
// The sweep thread has no request context, so it iterates every account's partition
// and deletes per account rather than through the current-account view.
diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java
index 0318a92096..dde1b6abc9 100644
--- a/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java
@@ -11,6 +11,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -25,6 +26,33 @@ class AtomicStorageBatchTest {
@TempDir
Path directory;
+ @ParameterizedTest
+ @ValueSource(strings = {"memory", "persistent", "hybrid", "wal"})
+ void atomicDeletionRemovesSelectedAccountsAndSurvivesRestart(String mode) {
+ StorageBackend<String, Integer> backend = backend(mode);
+ backend.load();
+ try {
+ AccountAwareStorageBackend<Integer> accounts = new AccountAwareStorageBackend<>(backend, null, "111111111111");
+ accounts.putAllAtomically(Map.of("a", 1, "b", 2));
+ accounts.putAllAtomicallyForAccount("222222222222", Map.of("a", 3, "keep", 4));
+ accounts.deleteAllAtomicallyForAccounts(Map.of("111111111111", Set.of("a", "b"), "222222222222", Set.of("a")));
+ assertEquals(Set.of("222222222222/keep"), backend.keys());
+ assertEquals(4, accounts.getForAccount("222222222222", "keep").orElseThrow());
+ backend.flush();
+ } finally {
+ close(backend);
+ }
+ if (!"memory".equals(mode)) {
+ StorageBackend<String, Integer> reopened = backend(mode);
+ reopened.load();
+ try {
+ assertEquals(Set.of("222222222222/keep"), reopened.keys());
+ } finally {
+ close(reopened);
+ }
+ }
+ }
+
@ParameterizedTest
@ValueSource(strings = {"memory", "persistent", "hybrid", "wal"})
void publishesWholeBatchWithAccountIsolation(String mode) {
@@ -61,6 +89,7 @@ class AtomicStorageBatchTest {
Files.writeString(snapshot.resolve("blocker"), "owned test file");
assertThrows(UncheckedIOException.class,
() -> store.putAllAtomically(Map.of("source", 1, "destination", 3)));
+ assertThrows(UncheckedIOException.class, () -> store.deleteAllAtomically(Set.of("source", "destination")));
assertEquals(3, store.get("source").orElseThrow());
assertEquals(1, store.get("destination").orElseThrow());
PersistentStorage<String, Integer> recovered = new PersistentStorage<>(backup, TYPE);
@@ -289,6 +318,7 @@ class AtomicStorageBatchTest {
Files.createDirectory(log);
assertThrows(UncheckedIOException.class,
() -> store.putAllAtomically(Map.of("source", 1, "destination", 3)));
+ assertThrows(UncheckedIOException.class, () -> store.deleteAllAtomically(Set.of("source", "destination")));
assertEquals(3, store.get("source").orElseThrow());
assertEquals(1, store.get("destination").orElseThrow());
Files.delete(log);
@@ -301,6 +331,39 @@ class AtomicStorageBatchTest {
}
}
+ @Test
+ void tornWalDeletionBatchNeverRemovesOnlySomeKeys() throws Exception {
+ Path snapshot = directory.resolve("data.json");
+ Path log = directory.resolve("data.wal");
+ WalStorage<String, Integer> writer = new WalStorage<>(snapshot, log, TYPE, Long.MAX_VALUE);
+ writer.load();
+ try {
+ writer.putAllAtomically(Map.of("a", 1, "b", 2, "keep", 3));
+ writer.flush();
+ byte[] initial = Files.readAllBytes(snapshot);
+ writer.deleteAllAtomically(Set.of("a", "b"));
+ byte[] complete = Files.readAllBytes(log);
+ for (int length = 0; length <= complete.length; length++) {
+ Path replaySnapshot = directory.resolve("replay-" + length + ".json");
+ Path replayLog = directory.resolve("replay-" + length + ".wal");
+ Files.write(replaySnapshot, initial);
+ Files.write(replayLog, Arrays.copyOf(complete, length));
+ WalStorage<String, Integer> replay = new WalStorage<>(replaySnapshot, replayLog, TYPE, Long.MAX_VALUE);
+ replay.load();
+ try {
+ boolean deleted = length == complete.length;
+ assertEquals(deleted, replay.get("a").isEmpty(), "prefix " + length);
+ assertEquals(deleted, replay.get("b").isEmpty(), "prefix " + length);
+ assertEquals(3, replay.get("keep").orElseThrow());
+ } finally {
+ replay.shutdown();
+ }
+ }
+ } finally {
+ writer.shutdown();
+ }
+ }
+
private StorageBackend<String, Integer> backend(String mode) {
return switch (mode) {
case "persistent" -> new PersistentStorage<>(directory.resolve("data.json"), TYPE);
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutorTest.java
index fef015e859..709269a096 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataExecutorTest.java
@@ -2,16 +2,130 @@ package io.github.hectorvent.floci.services.redshiftdata;
import io.github.hectorvent.floci.core.common.AwsException;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import java.sql.Statement;
import java.time.Duration;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
class RedshiftDataExecutorTest {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void shutdownDoesNotWaitForPendingAdmissionAndStillCompletesItAsCancelled(boolean queued) throws Exception {
+ try (RedshiftDataExecutor executor = new RedshiftDataExecutor(1, 1, 0);
+ ExecutorService callers = Executors.newFixedThreadPool(2)) {
+ CountDownLatch busy = new CountDownLatch(1);
+ CountDownLatch releaseBusy = new CountDownLatch(1);
+ CountDownLatch admitting = new CountDownLatch(1);
+ CountDownLatch releaseAdmission = new CountDownLatch(1);
+ AtomicBoolean ran = new AtomicBoolean();
+ AtomicBoolean cancelled = new AtomicBoolean();
+ if (queued) {
+ executor.submit("111111111111", "us-east-1", "busy", execution -> {
+ busy.countDown();
+ try {
+ releaseBusy.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException expected) {
+ // Shutdown interrupts this occupied worker before it can dequeue the reservation.
+ Thread.currentThread().interrupt();
+ }
+ });
+ assertTrue(busy.await(5, TimeUnit.SECONDS));
+ }
+ Future<RedshiftDataExecution> submission = callers.submit(() -> executor.submit(
+ "111111111111", "us-east-1", "pending", () -> {
+ admitting.countDown();
+ await(releaseAdmission);
+ }, execution -> {
+ ran.set(true);
+ cancelled.set(execution.isCancellationRequested());
+ }, (execution, failure) -> { }));
+ assertTrue(admitting.await(5, TimeUnit.SECONDS));
+ Future<?> shutdown = callers.submit(executor::close);
+ try {
+ shutdown.get(1, TimeUnit.SECONDS);
+ } finally {
+ releaseAdmission.countDown();
+ releaseBusy.countDown();
+ }
+ RedshiftDataExecution pending = submission.get(5, TimeUnit.SECONDS);
+ pending.await(Duration.ofSeconds(2));
+ assertTrue(pending.isDone(), "late admission must not strand a SUBMITTED execution");
+ assertTrue(ran.get());
+ assertTrue(cancelled.get());
+ }
+ }
+
+ @Test
+ void rejectedQueueDoesNotRunAdmissionWrites() throws Exception {
+ try (RedshiftDataExecutor executor = new RedshiftDataExecutor(1, 1, 1)) {
+ CountDownLatch started = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ try {
+ executor.submit("111111111111", "us-east-1", "running", execution -> {
+ started.countDown();
+ await(release);
+ });
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ executor.submit("111111111111", "us-east-1", "queued", execution -> { });
+ AtomicBoolean written = new AtomicBoolean();
+ assertThrows(AwsException.class, () -> executor.submit("111111111111", "us-east-1", "rejected",
+ () -> written.set(true), execution -> { }, (execution, failure) -> { }));
+ assertFalse(written.get(), "a rejected queue entry must not leave durable admission state");
+ } finally {
+ release.countDown();
+ }
+ }
+ }
+
+ @Test
+ void failingStorageResetStillSignalsJdbcCancellationAndCompletesQueuedWork() throws Exception {
+ try (RedshiftDataExecutor executor = new RedshiftDataExecutor(1, 1, 1)) {
+ Statement jdbc = mock(Statement.class);
+ CountDownLatch started = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ CountDownLatch cancelled = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ cancelled.countDown();
+ return null;
+ }).when(jdbc).cancel();
+ try {
+ executor.submit("111111111111", "us-east-1", "running", execution -> {
+ try {
+ execution.track(jdbc);
+ } catch (Exception failure) {
+ throw new AssertionError(failure);
+ }
+ started.countDown();
+ await(release);
+ });
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ AtomicBoolean queuedRan = new AtomicBoolean();
+ RedshiftDataExecution queued = executor.submit("111111111111", "us-east-1", "queued",
+ execution -> queuedRan.set(true));
+ IllegalStateException storageFailure = new IllegalStateException("receipt clear failed");
+ assertSame(storageFailure, assertThrows(IllegalStateException.class, () -> executor.clear(() -> {
+ throw storageFailure;
+ })));
+ assertTrue(cancelled.await(2, TimeUnit.SECONDS), "reset failure must not skip JDBC cancellation");
+ assertTrue(queued.isDone());
+ assertFalse(queuedRan.get());
+ } finally {
+ release.countDown();
+ }
+ }
+ }
+
@Test
void admissionReturnsWhileSqlIsRunningAndRejectsOverflow() throws Exception {
try (RedshiftDataExecutor executor = new RedshiftDataExecutor(1, 1, 1)) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyIntegrationTest.java
new file mode 100644
index 0000000000..0221425fa9
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyIntegrationTest.java
@@ -0,0 +1,146 @@
+package io.github.hectorvent.floci.services.redshiftdata;
+
+import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.InjectMock;
+import io.quarkus.test.common.http.TestHTTPResource;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.h2.Driver;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.redshiftdata.RedshiftDataClient;
+import software.amazon.awssdk.services.redshiftdata.model.BatchExecuteStatementRequest;
+import software.amazon.awssdk.services.redshiftdata.model.ExecuteStatementRequest;
+import software.amazon.awssdk.services.redshiftdata.model.ExecuteStatementResponse;
+import software.amazon.awssdk.services.redshiftdata.model.ResourceNotFoundException;
+import software.amazon.awssdk.services.redshiftdata.model.StatusString;
+
+import java.net.URI;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import java.time.Duration;
+import java.util.UUID;
+
+import static org.awaitility.Awaitility.await;
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+@QuarkusTest
+class RedshiftDataIdempotencyIntegrationTest {
+ @TestHTTPResource("/")
+ URI endpoint;
+
+ @InjectMock
+ RedshiftDataResourceResolver resolver;
+ @InjectMock
+ RedshiftDataConnectionFactory factory;
+ @Inject
+ RedshiftDataService service;
+ @Inject
+ RegionResolver regions;
+
+ private RedshiftDataClient data;
+
+ @BeforeEach
+ void setup() throws Exception {
+ Driver.load();
+ String jdbcUrl = "jdbc:h2:mem:" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1";
+ try (Connection connection = DriverManager.getConnection(jdbcUrl); Statement statement = connection.createStatement()) {
+ statement.execute("CREATE TABLE mutations (id INT)");
+ }
+ when(factory.open(any())).thenAnswer(invocation -> DriverManager.getConnection(jdbcUrl));
+ when(resolver.resolve(any(), any())).thenAnswer(invocation -> {
+ String arn = AwsArnUtils.Arn.of("redshift", invocation.getArgument(1), regions.getAccountId(), "cluster:wh").toString();
+ return new RedshiftDataResourceResolver.DatabaseTarget(arn, "localhost", 5439, "dev", "admin", "password");
+ });
+ data = RedshiftDataClient.builder().endpointOverride(endpoint).region(Region.US_EAST_1)
+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test"))).build();
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @AfterEach
+ void cleanup() {
+ service.clear();
+ if (data != null) {
+ data.close();
+ }
+ }
+
+ @Test
+ void sdkRetriesSingleAndWorkgroupStatementsWithoutRepeatingSql() {
+ for (boolean serverless : new boolean[] {false, true}) {
+ ExecuteStatementRequest.Builder builder = ExecuteStatementRequest.builder().database("dev")
+ .clientToken(UUID.randomUUID().toString()).sql("INSERT INTO mutations VALUES (1)");
+ if (serverless) {
+ builder.workgroupName("analytics");
+ } else {
+ builder.clusterIdentifier("wh").dbUser("admin");
+ }
+ ExecuteStatementRequest request = builder.build();
+ ExecuteStatementResponse first = data.executeStatement(request);
+ ExecuteStatementResponse retry = data.executeStatement(request);
+ assertEquals(first.id(), retry.id());
+ assertEquals(first.createdAt(), retry.createdAt());
+ finished(first.id());
+ }
+ assertEquals(2L, count());
+ }
+
+ @Test
+ void sdkRetriesBatchWithoutRepeatingEitherMutation() {
+ BatchExecuteStatementRequest request = BatchExecuteStatementRequest.builder()
+ .clusterIdentifier("wh").dbUser("admin").database("dev")
+ .clientToken(UUID.randomUUID().toString())
+ .sqls("INSERT INTO mutations VALUES (1)", "INSERT INTO mutations VALUES (2)").build();
+ String id = data.batchExecuteStatement(request).id();
+ assertEquals(id, data.batchExecuteStatement(request).id());
+ finished(id);
+ assertEquals(2L, count());
+ }
+
+ @Test
+ void malformedTokenReturnsAnAwsJsonValidationError() {
+ RestAssuredJsonUtils.awsAction("RedshiftData", "ExecuteStatement", """
+ {"ClusterIdentifier":"wh","DbUser":"admin","Database":"dev","Sql":"SELECT 1","ClientToken":12}
+ """).then().statusCode(400).body("__type", containsString("ValidationException"));
+ }
+
+ @Test
+ void sdkTokenReceiptsAreIsolatedByTheSigningAccount() {
+ ExecuteStatementRequest request = ExecuteStatementRequest.builder().clusterIdentifier("wh").dbUser("admin")
+ .database("dev").clientToken(UUID.randomUUID().toString()).sql("SELECT 1").build();
+ String first = data.executeStatement(request).id();
+ try (RedshiftDataClient other = RedshiftDataClient.builder().endpointOverride(endpoint).region(Region.US_EAST_1)
+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("111111111111", "test"))).build()) {
+ String second = other.executeStatement(request).id();
+ assertNotEquals(first, second);
+ assertEquals(second, other.executeStatement(request).id());
+ assertEquals(first, data.executeStatement(request).id());
+ assertThrows(ResourceNotFoundException.class, () -> data.describeStatement(r -> r.id(second)));
+ }
+ finished(first);
+ }
+
+ private long count() {
+ String id = data.executeStatement(r -> r.clusterIdentifier("wh").dbUser("admin").database("dev")
+ .sql("SELECT COUNT(*) FROM mutations")).id();
+ finished(id);
+ return data.getStatementResult(r -> r.id(id)).records().getFirst().getFirst().longValue();
+ }
+
+ private void finished(String id) {
+ await().atMost(Duration.ofSeconds(10)).untilAsserted(() ->
+ assertEquals(StatusString.FINISHED, data.describeStatement(r -> r.id(id)).status()));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyServiceTest.java
new file mode 100644
index 0000000000..1945666e56
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyServiceTest.java
@@ -0,0 +1,607 @@
+package io.github.hectorvent.floci.services.redshiftdata;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.PersistentStorage;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import org.h2.Driver;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+
+class RedshiftDataIdempotencyServiceTest {
+
+ @Test
+ void slowReceiptWriteDoesNotBlockOtherTokensOrTokenlessRequests() throws Exception {
+ try (Fixture fixture = new Fixture(); ExecutorService callers = Executors.newFixedThreadPool(3)) {
+ CountDownLatch writing = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ RedshiftDataIdempotencyStore.Scope scope = invocation.getArgument(0);
+ if (scope.token().equals("slow")) {
+ writing.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ }
+ return invocation.callRealMethod();
+ }).when(fixture.receipts).put(any(), any());
+ Future<ObjectNode> slow = callers.submit(() -> fixture.service.executeStatement(
+ fixture.single("slow", "SELECT 1"), "us-east-1"));
+ try {
+ assertTrue(writing.await(5, TimeUnit.SECONDS));
+ Future<ObjectNode> independent = callers.submit(() -> fixture.service.executeStatement(
+ fixture.single("independent", "SELECT 1"), "us-east-1"));
+ Future<ObjectNode> tokenless = callers.submit(() -> {
+ ObjectNode request = fixture.single("unused", "SELECT 1");
+ request.remove("ClientToken");
+ return fixture.service.executeStatement(request, "us-east-1");
+ });
+ assertFalse(independent.get(2, TimeUnit.SECONDS).path("Id").asText().isEmpty());
+ assertFalse(tokenless.get(2, TimeUnit.SECONDS).path("Id").asText().isEmpty());
+ } finally {
+ release.countDown();
+ }
+ slow.get(5, TimeUnit.SECONDS);
+ }
+ }
+
+ @Test
+ void rejectedQueueCannotLeaveAReceiptAfterCleanupFailureAndRestart(@TempDir Path directory) throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ Path receiptFile = directory.resolve("idempotency.json");
+ RedshiftDataIdempotencyStore receipts = spy(persistentReceipts(receiptFile, fixture.clock));
+ RedshiftDataService service = fixture.service(fixture.statements, receipts);
+ doThrow(new IllegalStateException("receipt deletion unavailable")).when(receipts).removeIfStatement(any(), any());
+ CountDownLatch started = new CountDownLatch(2);
+ CountDownLatch release = new CountDownLatch(1);
+ try {
+ for (int i = 0; i < 2; i++) {
+ fixture.executor.submit("000000000000", "us-east-1", "running-" + i, execution -> {
+ started.countDown();
+ try {
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(failure);
+ }
+ });
+ }
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ for (int i = 0; i < 16; i++) {
+ fixture.executor.submit("000000000000", "us-east-1", "queued-" + i, execution -> { });
+ }
+ assertThrows(AwsException.class, () -> service.executeStatement(
+ fixture.single("rejected-persistent", "INSERT INTO mutations VALUES (1)"), "us-east-1"));
+ RedshiftDataIdempotencyStore reopened = persistentReceipts(receiptFile, fixture.clock);
+ assertNull(reopened.find(new RedshiftDataIdempotencyStore.Scope("000000000000", "us-east-1",
+ "ExecuteStatement", "rejected-persistent")));
+ verifyNoInteractions(fixture.factory);
+ } finally {
+ release.countDown();
+ }
+ }
+ }
+
+ @Test
+ void executeRetryReturnsOriginalIdentityAndMutatesOnce() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ ObjectNode request = fixture.single("insert-token", "INSERT INTO mutations VALUES (1)");
+ ObjectNode first = fixture.service.executeStatement(request, "us-east-1");
+ ObjectNode retry = fixture.service.executeStatement(request, "us-east-1");
+ assertEquals(first.path("Id"), retry.path("Id"));
+ assertEquals(first.path("CreatedAt"), retry.path("CreatedAt"));
+ assertEquals(1, fixture.count());
+ verify(fixture.factory, times(1)).open(any());
+ }
+ }
+
+ @Test
+ void batchRetryMutatesOnceAndPublishesOneParentEvent() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ ObjectNode request = fixture.mapper.createObjectNode().put("ClientToken", "batch-token")
+ .put("WaitTimeSeconds", 5).put("WithEvent", true);
+ request.putArray("Sqls").add("INSERT INTO mutations VALUES (1)")
+ .add("INSERT INTO mutations VALUES (2)");
+ ObjectNode first = fixture.service.batchExecuteStatement(request, "us-east-1");
+ ObjectNode retry = fixture.service.batchExecuteStatement(request, "us-east-1");
+ assertEquals(first.path("Id"), retry.path("Id"));
+ assertEquals(2, fixture.count());
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ verify(fixture.publisher, times(1)).publish(any()));
+ verify(fixture.factory, times(1)).open(any());
+ }
+ }
+
+ @Test
+ void absentDifferentAndDifferentlyCasedTokensAreIndependent() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ List<String> ids = new ArrayList<>();
+ for (String token : List.of("one", "One", "two", "", "")) {
+ ObjectNode request = fixture.single(token, "INSERT INTO mutations VALUES (1)");
+ if (token.isEmpty()) {
+ request.remove("ClientToken");
+ }
+ ids.add(fixture.service.executeStatement(request, "us-east-1").path("Id").asText());
+ }
+ assertEquals(5, ids.stream().distinct().count());
+ assertEquals(5, fixture.count());
+ }
+ }
+
+ @Test
+ void suppliedMalformedTokensAreRejectedBeforeAdmission() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ for (String token : List.of("", "x".repeat(65))) {
+ AwsException failure = assertThrows(AwsException.class, () -> fixture.service.executeStatement(
+ fixture.single(token, "SELECT 1"), "us-east-1"));
+ assertEquals("ValidationException", failure.getErrorCode());
+ }
+ ObjectNode numeric = fixture.single("token", "SELECT 1").put("ClientToken", 123);
+ assertThrows(AwsException.class, () -> fixture.service.batchExecuteStatement(numeric, "us-east-1"));
+ ObjectNode nullToken = fixture.single("token", "SELECT 1").putNull("ClientToken");
+ assertThrows(AwsException.class, () -> fixture.service.executeStatement(nullToken, "us-east-1"));
+ verifyNoInteractions(fixture.factory);
+ }
+ }
+
+ @Test
+ void concurrentRetriesOfRunningWorkAdmitOneStatement() throws Exception {
+ try (Fixture fixture = new Fixture(); ExecutorService clients = Executors.newFixedThreadPool(8)) {
+ CountDownLatch opened = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ when(fixture.factory.open(any())).thenAnswer(invocation -> {
+ opened.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ return DriverManager.getConnection(fixture.jdbcUrl);
+ });
+ ObjectNode request = fixture.single("parallel", "INSERT INTO mutations VALUES (1)").put("WithEvent", true);
+ request.remove("WaitTimeSeconds");
+ try {
+ List<Future<String>> replies = new ArrayList<>();
+ for (int i = 0; i < 8; i++) {
+ replies.add(clients.submit(() -> fixture.service.executeStatement(request, "us-east-1")
+ .path("Id").asText()));
+ }
+ assertTrue(opened.await(5, TimeUnit.SECONDS));
+ List<String> ids = new ArrayList<>();
+ for (Future<String> reply : replies) {
+ ids.add(reply.get(5, TimeUnit.SECONDS));
+ }
+ assertEquals(1, ids.stream().distinct().count());
+ } finally {
+ release.countDown();
+ }
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertEquals(1, fixture.count()));
+ verify(fixture.factory, times(1)).open(any());
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> verify(fixture.publisher, times(1)).publish(any()));
+ }
+ }
+
+ @Test
+ void failedStatementReplaysUntilReset() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ ObjectNode request = fixture.single("failed", "SELECT * FROM missing_table");
+ String id = fixture.service.executeStatement(request, "us-east-1").path("Id").asText();
+ assertEquals(RedshiftDataStatementStore.Status.FAILED, fixture.statements.get(id).status);
+ assertEquals(id, fixture.service.executeStatement(request, "us-east-1").path("Id").asText());
+ fixture.service.clear();
+ assertNotEquals(id, fixture.service.executeStatement(request, "us-east-1").path("Id").asText());
+ verify(fixture.factory, times(2)).open(any());
+ }
+ }
+
+ @Test
+ void tokenScopeSeparatesRegionsAndExecuteOperations() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ String first = fixture.service.executeStatement(fixture.single("shared", "SELECT 1"), "us-east-1")
+ .path("Id").asText();
+ String otherRegion = fixture.service.executeStatement(fixture.single("shared", "SELECT 1"), "eu-west-1")
+ .path("Id").asText();
+ ObjectNode batch = fixture.mapper.createObjectNode().put("ClientToken", "shared").put("WaitTimeSeconds", 5);
+ batch.putArray("Sqls").add("SELECT 1");
+ String otherOperation = fixture.service.batchExecuteStatement(batch, "us-east-1").path("Id").asText();
+ assertEquals(3, List.of(first, otherRegion, otherOperation).stream().distinct().count());
+ }
+ }
+
+ @Test
+ void sameTokenReplaysOriginalPayloadAndDoesNotChangeEventPreference() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ ObjectNode first = fixture.service.executeStatement(
+ fixture.single("reused", "INSERT INTO mutations VALUES (1)"), "us-east-1");
+ ObjectNode retry = fixture.service.executeStatement(
+ fixture.single("reused", "INSERT INTO mutations VALUES (2)").put("WithEvent", true), "us-east-1");
+ assertEquals(first.path("Id"), retry.path("Id"));
+ assertEquals(1, fixture.count());
+ verifyNoInteractions(fixture.publisher);
+ }
+ }
+
+ @Test
+ void replayStillValidatesDatabaseAccess() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ ObjectNode request = fixture.single("access", "INSERT INTO mutations VALUES (1)");
+ fixture.service.executeStatement(request, "us-east-1");
+ when(fixture.resolver.resolve(any(), any())).thenThrow(new AwsException("ValidationException", "Access revoked", 400));
+ assertThrows(AwsException.class, () -> fixture.service.executeStatement(request, "us-east-1"));
+ verify(fixture.factory, times(1)).open(any());
+ }
+ }
+
+ @Test
+ void expiredTokenAdmitsNewWorkAndShortResultTtlDoesNotLoseTheReceipt() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ ObjectNode request = fixture.single("retention", "INSERT INTO mutations VALUES (1)");
+ ObjectNode first = fixture.service.executeStatement(request, "us-east-1");
+ fixture.clock.advance(Duration.ofHours(2));
+ fixture.statements.sweep();
+ assertNull(fixture.statements.get(first.path("Id").asText()));
+ assertEquals(first, fixture.service.executeStatement(request, "us-east-1"));
+ assertEquals(1, fixture.count());
+ fixture.clock.advance(Duration.ofHours(6));
+ assertNotEquals(first.path("Id"), fixture.service.executeStatement(request, "us-east-1").path("Id"));
+ assertEquals(2, fixture.count());
+ }
+ }
+
+ @Test
+ void rejectedAdmissionAndStorageFailuresLeaveTokenReusable() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ ObjectNode request = fixture.single("retry-error", "INSERT INTO mutations VALUES (1)");
+ doThrow(new IllegalStateException("receipt write failed")).doCallRealMethod()
+ .when(fixture.receipts).put(any(), any());
+ assertThrows(IllegalStateException.class, () -> fixture.service.executeStatement(request, "us-east-1"));
+ assertTrue(fixture.statements.values().isEmpty());
+ verifyNoInteractions(fixture.factory);
+ fixture.service.executeStatement(request, "us-east-1");
+ assertEquals(1, fixture.count());
+ }
+ }
+
+ @Test
+ void statementWriteFailureDoesNotConsumeToken() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ ObjectNode request = fixture.single("statement-write", "INSERT INTO mutations VALUES (1)");
+ doThrow(new IllegalStateException("statement write failed")).doCallRealMethod()
+ .when(fixture.statements).admitForAccount(any(), any());
+ assertThrows(IllegalStateException.class, () -> fixture.service.executeStatement(request, "us-east-1"));
+ verifyNoInteractions(fixture.factory);
+ fixture.service.executeStatement(request, "us-east-1");
+ assertEquals(1, fixture.count());
+ }
+ }
+
+ @Test
+ void queueFullReplaysAcceptedTokenAndRejectedTokenCanBeRetried() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ fixture.service.executeStatement(fixture.single("accepted", "INSERT INTO mutations VALUES (1)"), "us-east-1");
+ CountDownLatch started = new CountDownLatch(2);
+ CountDownLatch release = new CountDownLatch(1);
+ try {
+ for (int i = 0; i < 2; i++) {
+ fixture.executor.submit("000000000000", "us-east-1", "blocker-" + i, execution -> {
+ started.countDown();
+ try {
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(failure);
+ }
+ });
+ }
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ ObjectNode queued = fixture.single("queued", "INSERT INTO mutations VALUES (2)");
+ queued.remove("WaitTimeSeconds");
+ for (int i = 0; i < 16; i++) {
+ fixture.service.executeStatement(queued.deepCopy().put("ClientToken", "queued-" + i), "us-east-1");
+ }
+ ObjectNode rejected = fixture.single("rejected", "INSERT INTO mutations VALUES (3)");
+ rejected.remove("WaitTimeSeconds");
+ AwsException failure = assertThrows(AwsException.class,
+ () -> fixture.service.executeStatement(rejected, "us-east-1"));
+ assertEquals("ActiveStatementsExceededException", failure.getErrorCode());
+ fixture.service.executeStatement(fixture.single("accepted", "INSERT INTO mutations VALUES (1)"), "us-east-1");
+ } finally {
+ release.countDown();
+ }
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertEquals(17, fixture.count()));
+ fixture.service.executeStatement(fixture.single("rejected", "INSERT INTO mutations VALUES (3)"), "us-east-1");
+ assertEquals(18, fixture.count());
+ }
+ }
+
+ @Test
+ void restartRetainsReceiptAndFailsInterruptedWorkWithoutReexecutingSql(@TempDir Path directory) throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ Path receiptFile = directory.resolve("idempotency.json");
+ Path statementFile = directory.resolve("statements.json");
+ RedshiftDataIdempotencyStore originalReceipts = persistentReceipts(receiptFile, fixture.clock);
+ RedshiftDataStatementStore originalStatements = persistentStatements(statementFile, originalReceipts);
+ RedshiftDataService firstService = fixture.service(originalStatements, originalReceipts);
+ ObjectNode request = fixture.single("restart", "INSERT INTO mutations VALUES (1)");
+ String id = firstService.executeStatement(request, "us-east-1").path("Id").asText();
+ RedshiftDataStatementStore.StoredStatement interrupted = originalStatements.get(id);
+ interrupted.status = RedshiftDataStatementStore.Status.STARTED;
+ originalStatements.put(interrupted);
+ originalStatements.flush();
+
+ RedshiftDataIdempotencyStore reopenedReceipts = persistentReceipts(receiptFile, fixture.clock);
+ RedshiftDataStatementStore reopenedStatements = persistentStatements(statementFile, reopenedReceipts);
+ reopenedStatements.start();
+ try {
+ assertEquals(RedshiftDataStatementStore.Status.FAILED, reopenedStatements.get(id).status);
+ RedshiftDataService reopenedService = fixture.service(reopenedStatements, reopenedReceipts);
+ assertEquals(id, reopenedService.executeStatement(request, "us-east-1").path("Id").asText());
+ assertEquals(1, fixture.count());
+ verify(fixture.factory, times(1)).open(any());
+ verifyNoInteractions(fixture.publisher);
+ } finally {
+ reopenedStatements.stop();
+ }
+ }
+ }
+
+ @Test
+ void persistentStatementWriteFailureCannotAdmitSqlOrPublishAReceipt(@TempDir Path directory) throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ Path statementFile = directory.resolve("statements.json");
+ RedshiftDataStatementStore statements = persistentStatements(statementFile, fixture.receipts);
+ Files.createDirectory(statementFile);
+ Files.writeString(statementFile.resolve("blocked"), "cannot replace a nonempty directory");
+ RedshiftDataService service = fixture.service(statements, fixture.receipts);
+ assertThrows(UncheckedIOException.class,
+ () -> service.executeStatement(fixture.single("disk-failure", "INSERT INTO mutations VALUES (1)"), "us-east-1"));
+ verifyNoInteractions(fixture.factory);
+ verify(fixture.receipts, never()).put(any(), any());
+ assertTrue(statements.values().isEmpty());
+ }
+ }
+
+ @Test
+ void resetWaitsForAdmissionAndClearsItsReceipt() throws Exception {
+ try (Fixture fixture = new Fixture(); ExecutorService callers = Executors.newFixedThreadPool(2)) {
+ CountDownLatch writing = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ writing.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ return invocation.callRealMethod();
+ }).when(fixture.receipts).put(any(), any());
+ ObjectNode request = fixture.single("reset-race", "INSERT INTO mutations VALUES (1)");
+ request.remove("WaitTimeSeconds");
+ Future<ObjectNode> admitted = callers.submit(() -> fixture.service.executeStatement(request, "us-east-1"));
+ assertTrue(writing.await(5, TimeUnit.SECONDS));
+ Future<?> reset = callers.submit(fixture.service::clear);
+ try {
+ assertThrows(TimeoutException.class, () -> reset.get(100, TimeUnit.MILLISECONDS));
+ } finally {
+ release.countDown();
+ }
+ String originalId = admitted.get(5, TimeUnit.SECONDS).path("Id").asText();
+ reset.get(5, TimeUnit.SECONDS);
+ assertTrue(fixture.statements.values().isEmpty());
+ assertNotEquals(originalId, fixture.service.executeStatement(request, "us-east-1").path("Id").asText());
+ }
+ }
+
+ @Test
+ void cancelledQueuedStatementReplaysWithoutOpeningAConnection() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ CountDownLatch started = new CountDownLatch(2);
+ CountDownLatch release = new CountDownLatch(1);
+ try {
+ for (int i = 0; i < 2; i++) {
+ fixture.executor.submit("000000000000", "us-east-1", "blocker-" + i, execution -> {
+ started.countDown();
+ try {
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(failure);
+ }
+ });
+ }
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ ObjectNode request = fixture.single("cancelled", "INSERT INTO mutations VALUES (1)");
+ request.remove("WaitTimeSeconds");
+ String id = fixture.service.executeStatement(request, "us-east-1").path("Id").asText();
+ assertTrue(fixture.service.cancelStatement(fixture.mapper.createObjectNode().put("Id", id))
+ .path("Status").asBoolean());
+ assertEquals(RedshiftDataStatementStore.Status.ABORTED, fixture.statements.get(id).status);
+ assertEquals(id, fixture.service.executeStatement(request, "us-east-1").path("Id").asText());
+ verifyNoInteractions(fixture.factory);
+ } finally {
+ release.countDown();
+ }
+ }
+ }
+
+ @Test
+ void expiryDuringExecutionDoesNotLetOldCompletionReplaceTheNewReceipt() throws Exception {
+ try (Fixture fixture = new Fixture()) {
+ CountDownLatch opened = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicInteger connections = new AtomicInteger();
+ when(fixture.factory.open(any())).thenAnswer(invocation -> {
+ if (connections.getAndIncrement() == 0) {
+ opened.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ }
+ return DriverManager.getConnection(fixture.jdbcUrl);
+ });
+ ObjectNode request = fixture.single("expiry-active", "INSERT INTO mutations VALUES (1)");
+ request.remove("WaitTimeSeconds");
+ String original = fixture.service.executeStatement(request, "us-east-1").path("Id").asText();
+ assertTrue(opened.await(5, TimeUnit.SECONDS));
+ String replacement;
+ try {
+ fixture.clock.advance(Duration.ofHours(8));
+ replacement = fixture.service.executeStatement(request.deepCopy().put("WaitTimeSeconds", 5), "us-east-1")
+ .path("Id").asText();
+ assertNotEquals(original, replacement);
+ } finally {
+ release.countDown();
+ }
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertEquals(2, fixture.count()));
+ assertEquals(replacement, fixture.service.executeStatement(request, "us-east-1").path("Id").asText());
+ verify(fixture.factory, times(2)).open(any());
+ }
+ }
+
+ @Test
+ void waitingRetryDoesNotBlockAnotherTokensAdmission() throws Exception {
+ try (Fixture fixture = new Fixture(); ExecutorService callers = Executors.newFixedThreadPool(2)) {
+ CountDownLatch opened = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ CountDownLatch retryLookup = new CountDownLatch(1);
+ AtomicInteger connections = new AtomicInteger();
+ when(fixture.factory.open(any())).thenAnswer(invocation -> {
+ if (connections.getAndIncrement() == 0) {
+ opened.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ }
+ return DriverManager.getConnection(fixture.jdbcUrl);
+ });
+ ObjectNode request = fixture.single("waiting", "INSERT INTO mutations VALUES (1)");
+ request.remove("WaitTimeSeconds");
+ String first = fixture.service.executeStatement(request, "us-east-1").path("Id").asText();
+ assertTrue(opened.await(5, TimeUnit.SECONDS));
+ doAnswer(invocation -> {
+ retryLookup.countDown();
+ return invocation.callRealMethod();
+ }).when(fixture.receipts).find(any());
+ Future<ObjectNode> retry = callers.submit(() -> fixture.service.executeStatement(
+ request.deepCopy().put("WaitTimeSeconds", 5), "us-east-1"));
+ try {
+ assertTrue(retryLookup.await(5, TimeUnit.SECONDS));
+ Future<ObjectNode> independent = callers.submit(() -> fixture.service.executeStatement(
+ fixture.single("independent", "SELECT 1"), "us-east-1"));
+ assertNotEquals(first, independent.get(2, TimeUnit.SECONDS).path("Id").asText());
+ } finally {
+ release.countDown();
+ }
+ assertEquals(first, retry.get(5, TimeUnit.SECONDS).path("Id").asText());
+ }
+ }
+
+ private RedshiftDataIdempotencyStore persistentReceipts(Path file, Clock clock) {
+ PersistentStorage<String, RedshiftDataIdempotencyStore.Receipt> delegate = new PersistentStorage<>(file,
+ new TypeReference<Map<String, RedshiftDataIdempotencyStore.Receipt>>() { });
+ delegate.load();
+ return new RedshiftDataIdempotencyStore(new AccountAwareStorageBackend<>(delegate, null, "000000000000"), clock);
+ }
+
+ private RedshiftDataStatementStore persistentStatements(Path file, RedshiftDataIdempotencyStore receipts) {
+ PersistentStorage<String, RedshiftDataStatementStore.StoredStatement> delegate = new PersistentStorage<>(file,
+ new TypeReference<Map<String, RedshiftDataStatementStore.StoredStatement>>() { });
+ delegate.load();
+ StorageFactory factory = mock(StorageFactory.class);
+ when(factory.<RedshiftDataStatementStore.StoredStatement>create(eq("redshift-data"), eq("statements.json"), any()))
+ .thenReturn(new AccountAwareStorageBackend<>(delegate, null, "000000000000"));
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().redshiftData().resultTtlHours()).thenReturn(24);
+ return new RedshiftDataStatementStore(factory, config, receipts);
+ }
+
+ private static final class Fixture implements AutoCloseable {
+ private final ObjectMapper mapper = new ObjectMapper();
+ private final String jdbcUrl = "jdbc:h2:mem:" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1";
+ private final RedshiftDataConnectionFactory factory = mock(RedshiftDataConnectionFactory.class);
+ private final RedshiftDataEventPublisher publisher = mock(RedshiftDataEventPublisher.class);
+ private final RedshiftDataExecutor executor = new RedshiftDataExecutor(2, 16, 1);
+ private final MutableClock clock = new MutableClock();
+ private final RedshiftDataStatementStore statements = spy(new RedshiftDataStatementStore(1, clock));
+ private final RedshiftDataIdempotencyStore receipts = spy(new RedshiftDataIdempotencyStore(
+ AccountAwareStorageBackend.inMemory("000000000000"), clock));
+ private final RedshiftDataService service;
+ private final RedshiftDataResourceResolver resolver = mock(RedshiftDataResourceResolver.class);
+
+ private Fixture() throws Exception {
+ Driver.load();
+ try (Connection connection = DriverManager.getConnection(jdbcUrl); Statement statement = connection.createStatement()) {
+ statement.execute("CREATE TABLE mutations (id INT)");
+ }
+ when(resolver.resolve(any(), any())).thenReturn(new RedshiftDataResourceResolver.DatabaseTarget(
+ "arn:aws:redshift:us-east-1:000000000000:cluster:wh", "localhost", 5439, "dev", "admin", "password"));
+ when(factory.open(any())).thenAnswer(invocation -> DriverManager.getConnection(jdbcUrl));
+ service = service(statements, receipts);
+ }
+
+ private RedshiftDataService service(RedshiftDataStatementStore statements, RedshiftDataIdempotencyStore receipts) {
+ return new RedshiftDataService(resolver, factory, statements, mapper, null, executor, publisher,
+ null, null, receipts);
+ }
+
+ private ObjectNode single(String token, String sql) {
+ return mapper.createObjectNode().put("ClientToken", token).put("Sql", sql).put("WaitTimeSeconds", 5);
+ }
+
+ private int count() throws Exception {
+ try (Connection connection = DriverManager.getConnection(jdbcUrl);
+ Statement statement = connection.createStatement();
+ ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM mutations")) {
+ assertTrue(rows.next());
+ return rows.getInt(1);
+ }
+ }
+
+ @Override
+ public void close() {
+ executor.close();
+ }
+ }
+
+ private static final class MutableClock extends Clock {
+ private Instant now = Instant.now();
+
+ private void advance(Duration duration) {
+ now = now.plus(duration);
+ }
+
+ @Override
+ public ZoneId getZone() {
+ return ZoneOffset.UTC;
+ }
+
+ @Override
+ public Clock withZone(ZoneId zone) {
+ return Clock.fixed(now, zone);
+ }
+
+ @Override
+ public Instant instant() {
+ return now;
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStoreTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStoreTest.java
new file mode 100644
index 0000000000..bc9a6f3e16
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataIdempotencyStoreTest.java
@@ -0,0 +1,206 @@
+package io.github.hectorvent.floci.services.redshiftdata;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.HybridStorage;
+import io.github.hectorvent.floci.core.storage.PersistentStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.WalStorage;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+
+class RedshiftDataIdempotencyStoreTest {
+
+ private static final Instant START = Instant.parse("2026-10-23T00:00:00Z");
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ @Test
+ void expirySweepDoesNotRewriteTheFileForEveryReceipt(@TempDir Path directory) {
+ PersistentStorage<String, RedshiftDataIdempotencyStore.Receipt> delegate = spy(new PersistentStorage<>(
+ directory.resolve("idempotency.json"), new TypeReference<Map<String, RedshiftDataIdempotencyStore.Receipt>>() { }));
+ AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> backend =
+ new AccountAwareStorageBackend<>(delegate, null, "000000000000");
+ RedshiftDataIdempotencyStore initial = store(backend, START);
+ for (int i = 0; i < 25; i++) {
+ initial.put(scope(i % 2 == 0 ? "000000000000" : "111111111111", "us-east-1", "ExecuteStatement", "token-" + i),
+ mapper.createObjectNode().put("Id", "statement-" + i));
+ }
+ clearInvocations(delegate);
+ store(backend, START.plus(Duration.ofHours(8))).sweep();
+ assertTrue(backend.scanAllAccountEntries(key -> true).isEmpty());
+ verify(delegate, never()).delete(any());
+ verify(delegate, never()).flush();
+ verify(delegate, times(1)).deleteAllAtomically(any());
+ }
+
+ @Test
+ void sweepSnapshotDoesNotBlockNewReceiptsOrDeleteAReplacement() throws Exception {
+ AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> backend =
+ spy(AccountAwareStorageBackend.<RedshiftDataIdempotencyStore.Receipt>inMemory("000000000000"));
+ RedshiftDataIdempotencyStore.Scope scope = scope("000000000000", "us-east-1", "ExecuteStatement", "token");
+ store(backend, START).put(scope, mapper.createObjectNode().put("Id", "expired"));
+ RedshiftDataIdempotencyStore later = store(backend, START.plus(Duration.ofHours(8)));
+ CountDownLatch scanned = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ Object entries = invocation.callRealMethod();
+ scanned.countDown();
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ return entries;
+ }).when(backend).scanAllAccountEntries(any());
+ try (ExecutorService workers = Executors.newFixedThreadPool(2)) {
+ Future<?> sweep = workers.submit(later::sweep);
+ try {
+ assertTrue(scanned.await(5, TimeUnit.SECONDS));
+ Future<?> replacement = workers.submit(() -> later.put(scope, mapper.createObjectNode().put("Id", "replacement")));
+ replacement.get(2, TimeUnit.SECONDS);
+ } finally {
+ release.countDown();
+ }
+ sweep.get(5, TimeUnit.SECONDS);
+ assertEquals("replacement", later.find(scope).response.path("Id").asText());
+ }
+ }
+
+ @Test
+ void deadlineDoesNotSlideAndExpiresAtEightHours() {
+ AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> backend =
+ AccountAwareStorageBackend.inMemory("000000000000");
+ RedshiftDataIdempotencyStore initial = store(backend, START);
+ RedshiftDataIdempotencyStore.Scope scope = scope("000000000000", "us-east-1", "ExecuteStatement", "token");
+ initial.put(scope, mapper.createObjectNode().put("Id", "original"));
+ RedshiftDataIdempotencyStore later = store(backend, START.plus(Duration.ofHours(8)).minusNanos(1));
+ assertEquals("original", later.find(scope).response.path("Id").asText());
+ assertEquals(START.plus(Duration.ofHours(8)), later.find(scope).expiresAt);
+ assertNull(store(backend, START.plus(Duration.ofHours(8))).find(scope));
+ }
+
+ @Test
+ void scopesCannotCollideAndReceiptsAreIndependentSnapshots() {
+ RedshiftDataIdempotencyStore store = store(AccountAwareStorageBackend.inMemory("000000000000"), START);
+ RedshiftDataIdempotencyStore.Scope original = scope("000000000000", "us-east-1", "ExecuteStatement", "x:y/ z");
+ ObjectNode response = mapper.createObjectNode().put("Id", "original");
+ store.put(original, response);
+ response.put("Id", "mutated");
+ assertEquals("original", store.find(original).response.path("Id").asText());
+ store.find(original).response.put("Id", "mutated-read");
+ assertEquals("original", store.find(original).response.path("Id").asText());
+ assertNull(store.find(scope("111111111111", "us-east-1", "ExecuteStatement", "x:y/ z")));
+ assertNull(store.find(scope("000000000000", "eu-west-1", "ExecuteStatement", "x:y/ z")));
+ assertNull(store.find(scope("000000000000", "us-east-1", "BatchExecuteStatement", "x:y/ z")));
+ assertNull(store.find(scope("000000000000", "us-east-1", "ExecuteStatement", "X:y/ z")));
+ }
+
+ @Test
+ void staleRemovalCannotDeleteAReplacementReceipt() {
+ RedshiftDataIdempotencyStore store = store(AccountAwareStorageBackend.inMemory("000000000000"), START);
+ RedshiftDataIdempotencyStore.Scope scope = scope("000000000000", "us-east-1", "ExecuteStatement", "token");
+ store.put(scope, mapper.createObjectNode().put("Id", "new"));
+ store.removeIfStatement(scope, "old");
+ assertNotNull(store.find(scope));
+ store.removeIfStatement(scope, "new");
+ assertNull(store.find(scope));
+ }
+
+ @Test
+ void sweepAndClearCoverEveryAccount() {
+ AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> backend =
+ AccountAwareStorageBackend.inMemory("000000000000");
+ RedshiftDataIdempotencyStore store = store(backend, START);
+ store.put(scope("000000000000", "us-east-1", "ExecuteStatement", "a"), mapper.createObjectNode().put("Id", "a"));
+ store.put(scope("111111111111", "us-east-1", "ExecuteStatement", "a"), mapper.createObjectNode().put("Id", "b"));
+ store(backend, START.plus(Duration.ofHours(8))).sweep();
+ assertTrue(backend.scanAllAccountEntries(key -> true).isEmpty());
+ store.put(scope("111111111111", "us-east-1", "ExecuteStatement", "a"), mapper.createObjectNode().put("Id", "c"));
+ store.clear();
+ assertTrue(backend.scanAllAccountEntries(key -> true).isEmpty());
+ }
+
+ @Test
+ void receiptsSurvivePersistentBackendReopen(@TempDir Path directory) {
+ Path file = directory.resolve("idempotency.json");
+ RedshiftDataIdempotencyStore.Scope scope = scope("111111111111", "cn-north-1", "ExecuteStatement", "token");
+ RedshiftDataIdempotencyStore original = store(persistent(file), START);
+ assertNull(original.find(scope));
+ original.put(scope, mapper.createObjectNode().put("Id", "persisted").put("CreatedAt", 123.25));
+ RedshiftDataIdempotencyStore reopened = store(persistent(file), START.plusSeconds(60));
+ assertEquals("persisted", reopened.find(scope).response.path("Id").asText());
+ assertEquals(123.25, reopened.find(scope).response.path("CreatedAt").asDouble());
+ assertEquals(START.plus(Duration.ofHours(8)), reopened.find(scope).expiresAt);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"hybrid", "wal"})
+ void receiptsSurviveWriteBehindBackendRestart(String mode, @TempDir Path directory) {
+ RedshiftDataIdempotencyStore.Scope scope = scope("111111111111", "us-east-1", "ExecuteStatement", "token");
+ StorageBackend<String, RedshiftDataIdempotencyStore.Receipt> initial = writeBehind(mode, directory);
+ initial.load();
+ try {
+ store(new AccountAwareStorageBackend<>(initial, null, "000000000000"), START)
+ .put(scope, mapper.createObjectNode().put("Id", "persisted"));
+ } finally {
+ shutdown(initial);
+ }
+ StorageBackend<String, RedshiftDataIdempotencyStore.Receipt> reopened = writeBehind(mode, directory);
+ reopened.load();
+ try {
+ assertEquals("persisted", store(new AccountAwareStorageBackend<>(reopened, null, "000000000000"), START)
+ .find(scope).response.path("Id").asText());
+ } finally {
+ shutdown(reopened);
+ }
+ }
+
+ private StorageBackend<String, RedshiftDataIdempotencyStore.Receipt> writeBehind(String mode, Path directory) {
+ TypeReference<Map<String, RedshiftDataIdempotencyStore.Receipt>> type = new TypeReference<>() { };
+ return switch (mode) {
+ case "hybrid" -> new HybridStorage<>(directory.resolve("idempotency.json"), type, 60000);
+ case "wal" -> new WalStorage<>(directory.resolve("idempotency.json"), directory.resolve("idempotency.wal"), type, 60000);
+ default -> throw new IllegalArgumentException(mode);
+ };
+ }
+
+ private void shutdown(StorageBackend<String, RedshiftDataIdempotencyStore.Receipt> backend) {
+ if (backend instanceof HybridStorage<String, RedshiftDataIdempotencyStore.Receipt> hybrid) {
+ hybrid.shutdown();
+ } else if (backend instanceof WalStorage<String, RedshiftDataIdempotencyStore.Receipt> wal) {
+ wal.shutdown();
+ }
+ }
+
+ private AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> persistent(Path file) {
+ PersistentStorage<String, RedshiftDataIdempotencyStore.Receipt> delegate = new PersistentStorage<>(file,
+ new TypeReference<Map<String, RedshiftDataIdempotencyStore.Receipt>>() { });
+ delegate.load();
+ return new AccountAwareStorageBackend<>(delegate, null, "000000000000");
+ }
+
+ private RedshiftDataIdempotencyStore store(AccountAwareStorageBackend<RedshiftDataIdempotencyStore.Receipt> backend,
+ Instant now) {
+ return new RedshiftDataIdempotencyStore(backend, Clock.fixed(now, ZoneOffset.UTC));
+ }
+
+ private RedshiftDataIdempotencyStore.Scope scope(String account, String region, String operation, String token) {
+ return new RedshiftDataIdempotencyStore.Scope(account, region, operation, token);
+ }
+}