Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat: add DynamoDB to Redshift zero-ETL replication (#3872)

Vũ Thế Huy29b601834b3d
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index 68754bdf14..a5c1c779a3 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -50,6 +50,10 @@ DynamoDB Streams are supported via a separate target (`DynamoDBStreams_20120810`
| `GetShardIterator` | Get a shard iterator |
| `GetRecords` | Read stream records from a shard |
+Redshift zero-ETL integrations can consume these stream records directly. See the
+[Redshift DynamoDB zero-ETL](redshift.md#dynamodb-zero-etl) section for the supported target,
+landing table, checkpoint, and retry behavior.
+
## Configuration
| Variable | Default | Description |
diff --git a/docs/services/redshift.md b/docs/services/redshift.md
index 5ee4cd5cc1..6dac6438dd 100644
--- a/docs/services/redshift.md
+++ b/docs/services/redshift.md
@@ -33,7 +33,7 @@ For running SQL without a PostgreSQL wire connection (the way Lambda and Step Fu
| `DeleteTags` | Remove tags by key from a resource |
| `DescribeTags` | List tagged resources and their tags |
| `CreateClusterSubnetGroup` | Register a cluster subnet group (metadata only) |
-| `CreateIntegration` | Register a zero-ETL integration (metadata only; no data is replicated). Accepts `Description`, `KMSKeyId`, `AdditionalEncryptionContext` and `TagList` |
+| `CreateIntegration` | Create a DynamoDB Streams to provisioned Redshift zero-ETL integration. Accepts `Description`, `KMSKeyId`, `AdditionalEncryptionContext` and `TagList` |
| `DescribeIntegrations` | List integrations with `Filters`, `MaxRecords` and `Marker` pagination, or the one an `IntegrationArn` names |
| `DeleteIntegration` | Remove a zero-ETL integration |
| `DescribeClusterSubnetGroups` | List subnet groups, optionally filtered by name |
@@ -45,6 +45,21 @@ For running SQL without a PostgreSQL wire connection (the way Lambda and Step Fu
| `GetClusterCredentialsWithIAM` | Issue short-lived credentials with the DbUser derived from the caller's IAM identity |
<!-- floci:actions:end -->
+## DynamoDB zero-ETL
+
+The supported zero-ETL path is DynamoDB Streams to a provisioned Redshift cluster. The source ARN
+must identify an enabled local DynamoDB stream, and the target ARN must identify an existing
+provisioned Redshift cluster.
+
+Each stream record is written to a stable landing table named `floci_zetl_<integration-id>`.
+The landing table stores the event id, event name, source, region, sequence number, creation time,
+and DynamoDB keys and images as JSON text. Record event ids are unique, so retries are idempotent.
+
+The integration consumer has its own stream checkpoint and does not share Lambda event source
+mapping state. Deleting an integration stops its consumer while retaining the landing table.
+Serverless Redshift targets, backfill, schema inference, and relational projection are not supported
+in this first implementation.
+
## CloudFormation
Floci provisions these resource types:
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 8e67e37147..4d650105c0 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1396,6 +1396,9 @@ public interface EmulatorConfig {
@WithDefault("7199")
int proxyMaxPort();
+ @WithDefault("1000")
+ long pollIntervalMs();
+
// Hostname clients use to reach a cluster endpoint. Empty -> resolved from
// DockerHostResolver (falls back to "localhost").
Optional<String> endpointHost();
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java
new file mode 100644
index 0000000000..98d9bbd0ca
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java
@@ -0,0 +1,158 @@
+package io.github.hectorvent.floci.services.redshift;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;
+import io.github.hectorvent.floci.services.redshift.model.Integration;
+import io.github.hectorvent.floci.services.redshiftdata.RedshiftZeroEtlWriter;
+import io.quarkus.runtime.StartupEvent;
+import io.vertx.core.Vertx;
+import jakarta.annotation.PreDestroy;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@ApplicationScoped
+public class RedshiftDynamoDbZeroEtlConsumer {
+
+ private static final Logger LOG = Logger.getLogger(RedshiftDynamoDbZeroEtlConsumer.class);
+ private static final int BATCH_SIZE = 100;
+
+ private final Vertx vertx;
+ private final DynamoDbStreamService streamService;
+ private final RedshiftService redshiftService;
+ private final RedshiftZeroEtlWriter writer;
+ private final long pollIntervalMs;
+ private final ConcurrentHashMap<String, Long> timerIds = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, Boolean> activePolls = new ConcurrentHashMap<>();
+ private final ExecutorService pollExecutor = Executors.newCachedThreadPool(runnable -> {
+ Thread thread = new Thread(runnable, "redshift-zero-etl");
+ thread.setDaemon(true);
+ return thread;
+ });
+
+ @Inject
+ public RedshiftDynamoDbZeroEtlConsumer(Vertx vertx,
+ DynamoDbStreamService streamService,
+ RedshiftService redshiftService,
+ RedshiftZeroEtlWriter writer,
+ EmulatorConfig config) {
+ this(vertx, streamService, redshiftService, writer, config.services().redshift().pollIntervalMs());
+ }
+
+ RedshiftDynamoDbZeroEtlConsumer(DynamoDbStreamService streamService,
+ RedshiftService redshiftService,
+ RedshiftZeroEtlWriter writer) {
+ this(null, streamService, redshiftService, writer, 1000);
+ }
+
+ private RedshiftDynamoDbZeroEtlConsumer(Vertx vertx,
+ DynamoDbStreamService streamService,
+ RedshiftService redshiftService,
+ RedshiftZeroEtlWriter writer,
+ long pollIntervalMs) {
+ this.vertx = vertx;
+ this.streamService = streamService;
+ this.redshiftService = redshiftService;
+ this.writer = writer;
+ this.pollIntervalMs = pollIntervalMs;
+ }
+
+ void onStart(@Observes StartupEvent event) {
+ startPersistedIntegrations();
+ }
+
+ public void startPersistedIntegrations() {
+ for (Integration integration : redshiftService.listDynamoDbZeroEtlIntegrations()) {
+ if (integration.isPollingEnabled()) {
+ startPolling(integration);
+ }
+ }
+ }
+
+ public void startPolling(Integration integration) {
+ if (vertx == null || timerIds.containsKey(integration.getIntegrationArn())) {
+ return;
+ }
+ long timerId = vertx.setPeriodic(pollIntervalMs, ignored -> pollAnd(integration));
+ timerIds.put(integration.getIntegrationArn(), timerId);
+ }
+
+ public void stopPolling(String integrationArn) {
+ Long timerId = timerIds.remove(integrationArn);
+ if (timerId != null) {
+ vertx.cancelTimer(timerId);
+ }
+ activePolls.remove(integrationArn);
+ }
+
+ void pollOnce(Integration integration) {
+ writer.createLandingTable(integration.getAccountId(), integration.getTargetClusterIdentifier(),
+ integration.getLandingTableName());
+ String iteratorType = integration.getCheckpointSequenceNumber() == null
+ ? "TRIM_HORIZON" : "AFTER_SEQUENCE_NUMBER";
+ String iterator = streamService.getShardIterator(integration.getSourceStreamArn(),
+ DynamoDbStreamService.SHARD_ID, iteratorType, integration.getCheckpointSequenceNumber());
+ DynamoDbStreamService.GetRecordsResult result = streamService.getRecords(iterator, BATCH_SIZE);
+ if (result.records().isEmpty()) {
+ return;
+ }
+ String sequence = writer.writeBatch(integration.getAccountId(), integration.getTargetClusterIdentifier(),
+ integration.getLandingTableName(), result.records());
+ redshiftService.updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),
+ sequence, true, null);
+ integration.setCheckpointSequenceNumber(sequence);
+ }
+
+ public void reset() {
+ for (String integrationArn : timerIds.keySet()) {
+ stopPolling(integrationArn);
+ }
+ activePolls.clear();
+ }
+
+ @PreDestroy
+ void onStop() {
+ reset();
+ pollExecutor.shutdownNow();
+ }
+
+ private void pollAnd(Integration integration) {
+ String integrationArn = integration.getIntegrationArn();
+ if (activePolls.putIfAbsent(integrationArn, Boolean.TRUE) != null) {
+ return;
+ }
+ pollExecutor.submit(() -> {
+ try {
+ pollSafely(integration);
+ } finally {
+ activePolls.remove(integrationArn);
+ }
+ });
+ }
+
+ private void pollSafely(Integration integration) {
+ try {
+ pollOnce(integration);
+ } catch (Exception e) {
+ LOG.warnv(e, "Zero-ETL polling failed for integration {0}", integration.getIntegrationArn());
+ if (e instanceof AwsException awsException
+ && "TrimmedDataAccessException".equals(awsException.getErrorCode())) {
+ integration.setCheckpointSequenceNumber(null);
+ }
+ try {
+ redshiftService.updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),
+ integration.getCheckpointSequenceNumber(), false, e.getMessage());
+ } catch (Exception updateError) {
+ LOG.warnv(updateError, "Could not persist zero-ETL failure for integration {0}",
+ integration.getIntegrationArn());
+ }
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java
index 2ef5a99358..8dafed5def 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java
@@ -42,16 +42,25 @@ public class RedshiftQueryHandler {
private final EmulatorConfig config;
private final RedshiftIamDbUserResolver iamDbUserResolver;
private final RegionResolver regionResolver;
+ private final RedshiftDynamoDbZeroEtlConsumer zeroEtlConsumer;
@Inject
public RedshiftQueryHandler(RedshiftService service, RedshiftCredentialBroker credentialBroker,
EmulatorConfig config, RedshiftIamDbUserResolver iamDbUserResolver,
- RegionResolver regionResolver) {
+ RegionResolver regionResolver,
+ RedshiftDynamoDbZeroEtlConsumer zeroEtlConsumer) {
this.service = service;
this.credentialBroker = credentialBroker;
this.config = config;
this.iamDbUserResolver = iamDbUserResolver;
this.regionResolver = regionResolver;
+ this.zeroEtlConsumer = zeroEtlConsumer;
+ }
+
+ RedshiftQueryHandler(RedshiftService service, RedshiftCredentialBroker credentialBroker,
+ EmulatorConfig config, RedshiftIamDbUserResolver iamDbUserResolver,
+ RegionResolver regionResolver) {
+ this(service, credentialBroker, config, iamDbUserResolver, regionResolver, null);
}
public Response handle(String action, MultivaluedMap<String, String> params) {
@@ -372,6 +381,9 @@ public class RedshiftQueryHandler {
encryptionContextMap(params),
tagMap(params),
regionResolver.resolveRegionFromAuth(authorizationHeader));
+ if (zeroEtlConsumer != null) {
+ zeroEtlConsumer.startPolling(integration);
+ }
String xml = new XmlBuilder()
.start("CreateIntegrationResponse")
.start("CreateIntegrationResult")
@@ -414,6 +426,9 @@ public class RedshiftQueryHandler {
}
case "DeleteIntegration" -> {
Integration integration = service.deleteIntegration(params.getFirst("IntegrationArn"));
+ if (zeroEtlConsumer != null) {
+ zeroEtlConsumer.stopPolling(integration.getIntegrationArn());
+ }
String xml = new XmlBuilder()
.start("DeleteIntegrationResponse")
.start("DeleteIntegrationResult")
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
index 9dc645aa40..546ef96f14 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;
import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerHandle;
import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerManager;
@@ -74,6 +75,7 @@ public class RedshiftService {
private final RedshiftCredentialBroker credentialBroker;
private final SecretsManagerService secretsManagerService;
private final ObjectMapper objectMapper;
+ private final DynamoDbStreamService streamService;
// Proxy ports currently handed out, so allocateProxyPort never double-assigns within this JVM.
private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();
@@ -82,7 +84,8 @@ public class RedshiftService {
EmulatorConfig config, RegionResolver regionResolver,
RedshiftProxyManager proxyManager, DockerHostResolver dockerHostResolver,
RedshiftCredentialBroker credentialBroker,
- SecretsManagerService secretsManagerService, ObjectMapper objectMapper) {
+ SecretsManagerService secretsManagerService, ObjectMapper objectMapper,
+ DynamoDbStreamService streamService) {
this.clusters = storageFactory.create("redshift", "redshift-clusters.json", new TypeReference<Map<String, Cluster>>() {});
this.snapshots = storageFactory.create("redshift", "redshift-snapshots.json", new TypeReference<Map<String, Snapshot>>() {});
this.parameterGroups = storageFactory.create("redshift", "redshift-parameter-groups.json", new TypeReference<Map<String, ClusterParameterGroup>>() {});
@@ -96,6 +99,7 @@ public class RedshiftService {
this.credentialBroker = credentialBroker;
this.secretsManagerService = secretsManagerService;
this.objectMapper = objectMapper;
+ this.streamService = streamService;
}
RedshiftService(StorageFactory storageFactory, RedshiftContainerManager containerManager,
@@ -103,7 +107,7 @@ public class RedshiftService {
RedshiftProxyManager proxyManager, DockerHostResolver dockerHostResolver,
RedshiftCredentialBroker credentialBroker) {
this(storageFactory, containerManager, config, regionResolver, proxyManager, dockerHostResolver,
- credentialBroker, null, new ObjectMapper());
+ credentialBroker, null, new ObjectMapper(), null);
}
// Recreate Docker containers for persisted clusters on app restart (across every account, not just default)
@@ -314,6 +318,26 @@ public class RedshiftService {
if (targetArn == null || targetArn.isBlank()) {
throw new AwsException("InvalidParameterValue", "TargetArn is required.", 400);
}
+ AwsArnUtils.Arn source = parseZeroEtlArn(sourceArn, "DynamoDB stream");
+ if (!"dynamodb".equals(source.service()) || !source.resource().startsWith("table/")
+ || !source.resource().contains("/stream/")) {
+ throw new AwsException("InvalidParameterValue",
+ "SourceArn must identify a DynamoDB stream.", 400);
+ }
+ if (streamService == null) {
+ throw new AwsException("InternalFailure", "DynamoDB stream service is unavailable.", 500);
+ }
+ streamService.describeStream(sourceArn);
+ AwsArnUtils.Arn target = parseZeroEtlArn(targetArn, "Redshift cluster");
+ if (!"redshift".equals(target.service()) || !target.resource().startsWith("cluster:")) {
+ throw new AwsException("InvalidParameterValue",
+ "TargetArn must identify a provisioned Redshift cluster.", 400);
+ }
+ String clusterIdentifier = target.resource().substring("cluster:".length());
+ if (clusterIdentifier.isBlank()) {
+ throw new AwsException("InvalidParameterValue", "TargetArn must identify a Redshift cluster.", 400);
+ }
+ describeClusters(clusterIdentifier);
if (description != null && description.length() > MAX_INTEGRATION_DESCRIPTION) {
throw new AwsException("InvalidParameterValue",
"Description must be at most " + MAX_INTEGRATION_DESCRIPTION + " characters.", 400);
@@ -334,11 +358,19 @@ public class RedshiftService {
String integrationId = UUID.randomUUID().toString();
Integration integration = new Integration();
+ integration.setAccountId(integrations.accountId());
integration.setIntegrationArn("arn:aws:redshift:" + region + ":" + regionResolver.getAccountId()
+ ":integration:" + integrationId);
integration.setIntegrationName(integrationName);
integration.setSourceArn(sourceArn);
integration.setTargetArn(targetArn);
+ integration.setSourceStreamArn(sourceArn);
+ integration.setTargetClusterIdentifier(clusterIdentifier);
+ integration.setLandingTableName("floci_zetl_" + integrationId.replace('-', '_'));
+ integration.setCheckpointSequenceNumber(null);
+ integration.setRetryCount(0);
+ integration.setLastError(null);
+ integration.setPollingEnabled(true);
// Real integrations pass through creating before settling; nothing here has work to do.
integration.setStatus("active");
integration.setKmsKeyId(kmsKeyId);
@@ -351,6 +383,14 @@ public class RedshiftService {
return integration;
}
+ private static AwsArnUtils.Arn parseZeroEtlArn(String arn, String resourceType) {
+ try {
+ return AwsArnUtils.parse(arn);
+ } catch (IllegalArgumentException e) {
+ throw new AwsException("InvalidParameterValue", resourceType + " ARN is invalid.", 400);
+ }
+ }
+
/**
* Lists integrations with the documented filters and marker pagination.
*
@@ -401,6 +441,45 @@ public class RedshiftService {
return new IntegrationPage(List.copyOf(page), next);
}
+ public List<Integration> listDynamoDbZeroEtlIntegrations() {
+ return integrations.scanAllAccountEntries(key -> true).stream()
+ .filter(entry -> entry.value().getSourceStreamArn() != null
+ && !entry.value().getSourceStreamArn().isBlank())
+ .map(entry -> {
+ Integration integration = entry.value();
+ if (integration.getAccountId() == null) {
+ integration.setAccountId(entry.accountId());
+ }
+ return integration;
+ })
+ .toList();
+ }
+
+ public synchronized void updateIntegrationRuntime(String accountId, String integrationArn,
+ String checkpointSequenceNumber,
+ boolean successful, String error) {
+ for (String key : integrations.keysForAccount(accountId)) {
+ Optional<Integration> stored = integrations.getForAccount(accountId, key);
+ if (stored.isEmpty() || !integrationArn.equals(stored.get().getIntegrationArn())) {
+ continue;
+ }
+ Integration integration = stored.get();
+ if (successful) {
+ integration.setCheckpointSequenceNumber(checkpointSequenceNumber);
+ integration.setRetryCount(0);
+ integration.setLastError(null);
+ integration.setStatus("active");
+ } else {
+ integration.setRetryCount(integration.getRetryCount() + 1);
+ integration.setLastError(error);
+ integration.setStatus("failed");
+ }
+ integrations.putForAccount(accountId, key, integration);
+ return;
+ }
+ throw new AwsException("IntegrationNotFoundFault", "The requested integration doesn't exist.", 404);
+ }
+
/** One page of integrations plus the marker to continue with, or {@code null} at the end. */
public record IntegrationPage(List<Integration> integrations, String marker) {}
@@ -468,6 +547,17 @@ public class RedshiftService {
return clusters.scan(k -> true);
}
+ public List<Cluster> describeClustersForAccount(String accountId, String identifier) {
+ if (identifier != null) {
+ Optional<Cluster> cluster = clusters.getForAccount(accountId, identifier);
+ if (cluster.isEmpty()) {
+ throw new AwsException("ClusterNotFound", "Cluster " + identifier + " not found", 404);
+ }
+ return List.of(cluster.get());
+ }
+ return clusters.scanForAccount(accountId, k -> true);
+ }
+
public synchronized Cluster deleteCluster(String identifier) {
Optional<Cluster> clusterOpt = clusters.get(identifier);
if (clusterOpt.isEmpty()) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/model/Integration.java b/src/main/java/io/github/hectorvent/floci/services/redshift/model/Integration.java
index 6edf08a41c..085b35b75e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/model/Integration.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/model/Integration.java
@@ -16,9 +16,17 @@ import java.util.Map;
@RegisterForReflection
public class Integration {
private String integrationArn;
+ private String accountId;
private String integrationName;
private String sourceArn;
private String targetArn;
+ private String sourceStreamArn;
+ private String targetClusterIdentifier;
+ private String landingTableName;
+ private String checkpointSequenceNumber;
+ private int retryCount;
+ private String lastError;
+ private boolean pollingEnabled;
private String description;
private String status;
private String kmsKeyId;
@@ -31,12 +39,28 @@ public class Integration {
public String getIntegrationArn() { return integrationArn; }
public void setIntegrationArn(String integrationArn) { this.integrationArn = integrationArn; }
+ public String getAccountId() { return accountId; }
+ public void setAccountId(String accountId) { this.accountId = accountId; }
public String getIntegrationName() { return integrationName; }
public void setIntegrationName(String integrationName) { this.integrationName = integrationName; }
public String getSourceArn() { return sourceArn; }
public void setSourceArn(String sourceArn) { this.sourceArn = sourceArn; }
public String getTargetArn() { return targetArn; }
public void setTargetArn(String targetArn) { this.targetArn = targetArn; }
+ public String getSourceStreamArn() { return sourceStreamArn; }
+ public void setSourceStreamArn(String sourceStreamArn) { this.sourceStreamArn = sourceStreamArn; }
+ public String getTargetClusterIdentifier() { return targetClusterIdentifier; }
+ public void setTargetClusterIdentifier(String targetClusterIdentifier) { this.targetClusterIdentifier = targetClusterIdentifier; }
+ public String getLandingTableName() { return landingTableName; }
+ public void setLandingTableName(String landingTableName) { this.landingTableName = landingTableName; }
+ public String getCheckpointSequenceNumber() { return checkpointSequenceNumber; }
+ public void setCheckpointSequenceNumber(String checkpointSequenceNumber) { this.checkpointSequenceNumber = checkpointSequenceNumber; }
+ public int getRetryCount() { return retryCount; }
+ public void setRetryCount(int retryCount) { this.retryCount = retryCount; }
+ public String getLastError() { return lastError; }
+ public void setLastError(String lastError) { this.lastError = lastError; }
+ public boolean isPollingEnabled() { return pollingEnabled; }
+ public void setPollingEnabled(boolean pollingEnabled) { this.pollingEnabled = pollingEnabled; }
public String getDescription() { return description; }
public void setDescription(String description) { this.description = description; }
public String getStatus() { return status; }
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriter.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriter.java
new file mode 100644
index 0000000000..8d7cb06912
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriter.java
@@ -0,0 +1,147 @@
+package io.github.hectorvent.floci.services.redshiftdata;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;
+import io.github.hectorvent.floci.services.redshift.RedshiftService;
+import io.github.hectorvent.floci.services.redshift.model.Cluster;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Types;
+import java.time.Instant;
+import java.util.List;
+import java.util.regex.Pattern;
+
+@ApplicationScoped
+public class RedshiftZeroEtlWriter {
+
+ private static final Pattern TABLE_NAME = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$");
+ private static final String LANDING_TABLE_DDL = "CREATE TABLE IF NOT EXISTS %s ("
+ + "event_id TEXT PRIMARY KEY, "
+ + "event_name TEXT NOT NULL, "
+ + "event_source TEXT, "
+ + "aws_region TEXT, "
+ + "sequence_number TEXT NOT NULL, "
+ + "approximate_creation_datetime TIMESTAMPTZ, "
+ + "keys_json TEXT, "
+ + "old_image_json TEXT, "
+ + "new_image_json TEXT, "
+ + "ingested_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP";
+ private static final String INSERT_RECORD = "INSERT INTO %s (event_id, event_name, event_source, aws_region, "
+ + "sequence_number, approximate_creation_datetime, keys_json, old_image_json, new_image_json) "
+ + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (event_id) DO NOTHING";
+
+ private final RedshiftService redshiftService;
+ private final RedshiftDataConnectionFactory connectionFactory;
+ private final ObjectMapper objectMapper;
+
+ @Inject
+ RedshiftZeroEtlWriter(RedshiftService redshiftService,
+ RedshiftDataConnectionFactory connectionFactory,
+ ObjectMapper objectMapper) {
+ this.redshiftService = redshiftService;
+ this.connectionFactory = connectionFactory;
+ this.objectMapper = objectMapper;
+ }
+
+ public void createLandingTable(String accountId, String clusterIdentifier, String tableName) {
+ String safeTableName = validateTableName(tableName);
+ try (Connection connection = connectionFactory.open(target(accountId, clusterIdentifier));
+ Statement statement = connection.createStatement()) {
+ statement.executeUpdate(LANDING_TABLE_DDL.formatted(safeTableName) + ")");
+ } catch (SQLException e) {
+ throw new AwsException("InternalFailure", "Could not create Redshift zero-ETL landing table.", 500);
+ }
+ }
+
+ public String writeBatch(String accountId, String clusterIdentifier, String tableName,
+ List<DynamoDbStreamRecord> records) {
+ if (records == null || records.isEmpty()) {
+ return null;
+ }
+ String safeTableName = validateTableName(tableName);
+ String newestSequence = null;
+ try (Connection connection = connectionFactory.open(target(accountId, clusterIdentifier));
+ PreparedStatement statement = connection.prepareStatement(INSERT_RECORD.formatted(safeTableName))) {
+ for (DynamoDbStreamRecord record : records) {
+ bind(statement, record);
+ statement.executeUpdate();
+ newestSequence = record.getSequenceNumber();
+ }
+ } catch (SQLException e) {
+ throw new AwsException("InternalFailure", "Could not write Redshift zero-ETL records.", 500);
+ }
+ return newestSequence;
+ }
+
+ private void bind(PreparedStatement statement, DynamoDbStreamRecord record) throws SQLException {
+ statement.setString(1, record.getEventId());
+ statement.setString(2, record.getEventName());
+ setNullableString(statement, 3, record.getEventSource());
+ setNullableString(statement, 4, record.getAwsRegion());
+ statement.setString(5, record.getSequenceNumber());
+ if (record.getApproximateCreationDateTime() > 0) {
+ statement.setObject(6, Instant.ofEpochSecond(record.getApproximateCreationDateTime()));
+ } else {
+ statement.setNull(6, Types.TIMESTAMP_WITH_TIMEZONE);
+ }
+ setJson(statement, 7, record.getKeys());
+ setJson(statement, 8, record.getOldImage());
+ setJson(statement, 9, record.getNewImage());
+ }
+
+ private void setNullableString(PreparedStatement statement, int index, String value) throws SQLException {
+ if (value == null) {
+ statement.setNull(index, Types.VARCHAR);
+ } else {
+ statement.setString(index, value);
+ }
+ }
+
+ private void setJson(PreparedStatement statement, int index, JsonNode value) throws SQLException {
+ if (value == null || value.isNull()) {
+ statement.setNull(index, Types.VARCHAR);
+ return;
+ }
+ try {
+ statement.setString(index, objectMapper.writeValueAsString(value));
+ } catch (JsonProcessingException e) {
+ throw new AwsException("InternalFailure", "Could not serialize a DynamoDB stream record.", 500);
+ }
+ }
+
+ private RedshiftDataResourceResolver.DatabaseTarget target(String accountId, String clusterIdentifier) {
+ Cluster cluster;
+ try {
+ cluster = redshiftService.describeClustersForAccount(accountId, clusterIdentifier).get(0);
+ } catch (RuntimeException e) {
+ throw new AwsException("ResourceNotFoundException",
+ "Redshift cluster " + clusterIdentifier + " was not found.", 404);
+ }
+ if (cluster.getContainerHost() == null || cluster.getContainerHost().isBlank()
+ || cluster.getContainerPort() <= 0) {
+ throw new AwsException("InvalidClusterState", "Redshift cluster runtime is not available.", 400);
+ }
+ return new RedshiftDataResourceResolver.DatabaseTarget(
+ "cluster:" + clusterIdentifier,
+ cluster.getContainerHost(),
+ cluster.getContainerPort(),
+ "dev",
+ cluster.getMasterUsername(),
+ cluster.getMasterPassword());
+ }
+
+ private static String validateTableName(String tableName) {
+ if (tableName == null || !TABLE_NAME.matcher(tableName).matches()) {
+ throw new AwsException("InvalidParameterValue", "Landing table name is invalid.", 400);
+ }
+ return tableName;
+ }
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index ac4989b3cb..b988327aef 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -385,6 +385,7 @@ floci:
enabled: true
default-port: 5439
image-version: postgres:15-alpine
+ poll-interval-ms: 1000
proxy-base-port: 7100
proxy-max-port: 7199
default-credential-duration-seconds: 900 # GetClusterCredentials default lifetime (900-3600)
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java
new file mode 100644
index 0000000000..e59b89545b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java
@@ -0,0 +1,54 @@
+package io.github.hectorvent.floci.services.redshift;
+
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;
+import io.github.hectorvent.floci.services.redshift.model.Integration;
+import io.github.hectorvent.floci.services.redshiftdata.RedshiftZeroEtlWriter;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.*;
+
+class RedshiftDynamoDbZeroEtlConsumerTest {
+
+ @Test
+ void writesRecordsAndAdvancesCheckpointOnlyAfterSuccessfulBatch() {
+ DynamoDbStreamService streamService = mock(DynamoDbStreamService.class);
+ RedshiftService redshiftService = mock(RedshiftService.class);
+ RedshiftZeroEtlWriter writer = mock(RedshiftZeroEtlWriter.class);
+ Integration integration = integration();
+ DynamoDbStreamRecord record = new DynamoDbStreamRecord();
+ record.setEventId("event-1");
+ record.setSequenceNumber("000000000000000000001");
+ String iterator = "iterator";
+ when(streamService.getShardIterator(eq(integration.getSourceStreamArn()), eq(DynamoDbStreamService.SHARD_ID),
+ eq("TRIM_HORIZON"), eq(null))).thenReturn(iterator);
+ when(streamService.getRecords(iterator, 100)).thenReturn(
+ new DynamoDbStreamService.GetRecordsResult(List.of(record), "next"));
+ when(writer.writeBatch(integration.getAccountId(), "warehouse", "floci_zetl_orders", List.of(record)))
+ .thenReturn(record.getSequenceNumber());
+
+ RedshiftDynamoDbZeroEtlConsumer consumer =
+ new RedshiftDynamoDbZeroEtlConsumer(streamService, redshiftService, writer);
+ consumer.pollOnce(integration);
+
+ verify(writer).createLandingTable(integration.getAccountId(), "warehouse", "floci_zetl_orders");
+ verify(writer).writeBatch(integration.getAccountId(), "warehouse", "floci_zetl_orders", List.of(record));
+ verify(redshiftService).updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),
+ record.getSequenceNumber(), true, null);
+ }
+
+ private static Integration integration() {
+ Integration integration = new Integration();
+ integration.setIntegrationArn("arn:aws:redshift:us-east-1:111111111111:integration:one");
+ integration.setAccountId("111111111111");
+ integration.setSourceStreamArn("arn:aws:dynamodb:us-east-1:111111111111:table/orders/stream/one");
+ integration.setTargetClusterIdentifier("warehouse");
+ integration.setLandingTableName("floci_zetl_orders");
+ integration.setPollingEnabled(true);
+ return integration;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftIntegrationsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftIntegrationsIntegrationTest.java
index f8e58675c3..1845b4a979 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftIntegrationsIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftIntegrationsIntegrationTest.java
@@ -2,9 +2,14 @@ package io.github.hectorvent.floci.services.redshift;
import java.util.ArrayList;
import java.util.List;
+import io.restassured.RestAssured;
+import io.restassured.response.Response;
import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.config.HttpClientConfig;
+import io.restassured.config.RestAssuredConfig;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.restassured.specification.RequestSpecification;
-import io.restassured.response.Response;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static io.restassured.RestAssured.given;
@@ -26,9 +31,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
@QuarkusTest
class RedshiftIntegrationsIntegrationTest {
- private static final String SOURCE = "arn:aws:dynamodb:us-east-1:000000000000:table/keystone-main";
- private static final String TARGET =
- "arn:aws:redshift-serverless:us-east-1:000000000000:namespace/8445f0c7-d2b1-4c1c-916c-0eeaa68fd487";
+ private static String source;
+ private static String otherSource;
+ private static final String TARGET = "arn:aws:redshift:us-east-1:000000000000:cluster:zero-etl-cluster";
/**
* The Authorization header is what routes a Query request to a service; without it the
@@ -48,9 +53,50 @@ class RedshiftIntegrationsIntegrationTest {
return spec.when().post("/");
}
+ @BeforeEach
+ void createZeroEtlResources() {
+ if (source != null) {
+ return;
+ }
+ RestAssured.config = RestAssuredConfig.config()
+ .httpClient(HttpClientConfig.httpClientConfig().setParam("http.socket.timeout", 180_000));
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ String newSource = createDynamoTable("keystone-main");
+ String newOtherSource = createDynamoTable("keystone-other");
+ query("Action", "CreateCluster", "ClusterIdentifier", "zero-etl-cluster",
+ "NodeType", "dc2.large", "MasterUsername", "admin", "MasterUserPassword", "password123")
+ .then().statusCode(200);
+ source = newSource;
+ otherSource = newOtherSource;
+ }
+
+ private static String createDynamoTable(String tableName) {
+ Response response = given()
+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")
+ .contentType("application/x-amz-json-1.0")
+ .body("""
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName":"id","KeyType":"HASH"}],
+ "AttributeDefinitions": [{"AttributeName":"id","AttributeType":"S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}
+ }
+ """.formatted(tableName))
+ .when().post("/");
+ if (response.statusCode() != 200) {
+ response = given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeTable")
+ .contentType("application/x-amz-json-1.0")
+ .body("{\"TableName\":\"%s\"}".formatted(tableName))
+ .when().post("/");
+ }
+ return response.then().statusCode(200).extract().path("TableDescription.LatestStreamArn");
+ }
+
private static String createIntegration(String name) {
return query("Action", "CreateIntegration", "IntegrationName", name,
- "SourceArn", SOURCE, "TargetArn", TARGET)
+ "SourceArn", source, "TargetArn", TARGET)
.then().statusCode(200)
.extract().body().asString();
}
@@ -67,7 +113,7 @@ class RedshiftIntegrationsIntegrationTest {
query("Action", "DescribeIntegrations")
.then().statusCode(200)
.body(containsString("zetl-described"))
- .body(containsString(SOURCE))
+ .body(containsString(source))
.body(containsString(TARGET))
// Lower case on real Redshift, not ACTIVE.
.body(containsString("<Status>active</Status>"))
@@ -119,7 +165,7 @@ class RedshiftIntegrationsIntegrationTest {
createIntegration("zetl-duplicate");
query("Action", "CreateIntegration", "IntegrationName", "zetl-duplicate",
- "SourceArn", SOURCE, "TargetArn", TARGET)
+ "SourceArn", source, "TargetArn", TARGET)
.then().statusCode(400)
.body(containsString("IntegrationAlreadyExistsFault"));
}
@@ -132,7 +178,7 @@ class RedshiftIntegrationsIntegrationTest {
// trailing hyphen are each outside it, so AWS refuses names floci used to accept.
for (String name : new String[] {"1zetl", "-zetl", "zetl_name", "zetl--name", "zetl-"}) {
query("Action", "CreateIntegration", "IntegrationName", name,
- "SourceArn", SOURCE, "TargetArn", TARGET)
+ "SourceArn", source, "TargetArn", TARGET)
.then().statusCode(400)
.body(containsString("InvalidParameterValue"))
.body(containsString("IntegrationName"));
@@ -142,7 +188,7 @@ class RedshiftIntegrationsIntegrationTest {
@Test
void anIntegrationNameOverTheModelledLengthIsRejected() {
query("Action", "CreateIntegration", "IntegrationName", "z".repeat(64),
- "SourceArn", SOURCE, "TargetArn", TARGET)
+ "SourceArn", source, "TargetArn", TARGET)
.then().statusCode(400)
.body(containsString("InvalidParameterValue"));
}
@@ -166,7 +212,7 @@ class RedshiftIntegrationsIntegrationTest {
void tagsSurviveTheRoundTrip() {
// The Query member is TagList, not Tags: an SDK serialises the list under its own name.
query("Action", "CreateIntegration", "IntegrationName", "zetl-tagged",
- "SourceArn", SOURCE, "TargetArn", TARGET,
+ "SourceArn", source, "TargetArn", TARGET,
"TagList.Tag.1.Key", "Environment", "TagList.Tag.1.Value", "dev")
.then().statusCode(200);
@@ -187,7 +233,7 @@ class RedshiftIntegrationsIntegrationTest {
@Test
void descriptionAndEncryptionContextAreStoredAndReturned() {
query("Action", "CreateIntegration", "IntegrationName", "zetl-full",
- "SourceArn", SOURCE, "TargetArn", TARGET,
+ "SourceArn", source, "TargetArn", TARGET,
"Description", "nightly replica",
"KMSKeyId", "arn:aws:kms:us-east-1:000000000000:key/abc",
"AdditionalEncryptionContext.entry.1.key", "team",
@@ -204,7 +250,7 @@ class RedshiftIntegrationsIntegrationTest {
@Test
void encryptionContextWithoutAKmsKeyIsRejected() {
query("Action", "CreateIntegration", "IntegrationName", "zetl-nokms",
- "SourceArn", SOURCE, "TargetArn", TARGET,
+ "SourceArn", source, "TargetArn", TARGET,
"AdditionalEncryptionContext.entry.1.key", "team",
"AdditionalEncryptionContext.entry.1.value", "data")
.then().statusCode(400)
@@ -218,7 +264,7 @@ class RedshiftIntegrationsIntegrationTest {
// rather than absolute counts.
for (int i = 0; i < 21; i++) {
query("Action", "CreateIntegration", "IntegrationName", String.format("zetl-page-%02d", i),
- "SourceArn", SOURCE + "-" + i, "TargetArn", TARGET)
+ "SourceArn", source, "TargetArn", TARGET)
.then().statusCode(200);
}
@@ -283,15 +329,15 @@ class RedshiftIntegrationsIntegrationTest {
@Test
void filteringBySourceArnNarrowsTheResult() {
query("Action", "CreateIntegration", "IntegrationName", "zetl-filtered",
- "SourceArn", SOURCE + "-filtered", "TargetArn", TARGET)
+ "SourceArn", otherSource, "TargetArn", TARGET)
.then().statusCode(200);
query("Action", "CreateIntegration", "IntegrationName", "zetl-unfiltered",
- "SourceArn", SOURCE + "-other", "TargetArn", TARGET)
+ "SourceArn", source, "TargetArn", TARGET)
.then().statusCode(200);
query("Action", "DescribeIntegrations",
"Filters.DescribeIntegrationsFilter.1.Name", "source-arn",
- "Filters.DescribeIntegrationsFilter.1.Values.Value.1", SOURCE + "-filtered")
+ "Filters.DescribeIntegrationsFilter.1.Values.Value.1", otherSource)
.then().statusCode(200)
.body(containsString("zetl-filtered"))
.body(not(containsString("zetl-unfiltered")));
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
index a428de1adf..871f629dcd 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
@@ -6,12 +6,15 @@ import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
+import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;
import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerHandle;
import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerManager;
import io.github.hectorvent.floci.services.redshift.model.Cluster;
import io.github.hectorvent.floci.services.redshift.model.ClusterParameterGroup;
import io.github.hectorvent.floci.services.redshift.model.ClusterSubnetGroup;
import io.github.hectorvent.floci.services.redshift.model.Endpoint;
+import io.github.hectorvent.floci.services.redshift.model.Integration;
import io.github.hectorvent.floci.services.redshift.model.Parameter;
import io.github.hectorvent.floci.services.redshift.model.Snapshot;
import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;
@@ -45,12 +48,14 @@ class RedshiftServiceTest {
private AccountAwareStorageBackend<String> snapshotDumpBackend;
private AccountAwareStorageBackend<ClusterParameterGroup> parameterGroupBackend;
private AccountAwareStorageBackend<ClusterSubnetGroup> subnetGroupBackend;
+ private AccountAwareStorageBackend<Integration> integrationBackend;
private RedshiftContainerManager cm;
private RegionResolver regionResolver;
private RedshiftProxyManager proxyManager;
private DockerHostResolver dockerHostResolver;
private RedshiftCredentialBroker credentialBroker;
private SecretsManagerService secretsManagerService;
+ private DynamoDbStreamService streamService;
private RedshiftService service;
@BeforeEach
@@ -62,6 +67,7 @@ class RedshiftServiceTest {
snapshotDumpBackend = mock(AccountAwareStorageBackend.class);
parameterGroupBackend = mock(AccountAwareStorageBackend.class);
subnetGroupBackend = mock(AccountAwareStorageBackend.class);
+ integrationBackend = mock(AccountAwareStorageBackend.class);
cm = mock(RedshiftContainerManager.class);
proxyManager = mock(RedshiftProxyManager.class);
dockerHostResolver = mock(DockerHostResolver.class);
@@ -86,15 +92,49 @@ class RedshiftServiceTest {
when(sf.<Snapshot>create(eq("redshift"), eq("redshift-snapshots.json"), any())).thenReturn(snapshotBackend);
when(sf.<ClusterParameterGroup>create(eq("redshift"), eq("redshift-parameter-groups.json"), any())).thenReturn(parameterGroupBackend);
when(sf.<ClusterSubnetGroup>create(eq("redshift"), eq("redshift-subnet-groups.json"), any())).thenReturn(subnetGroupBackend);
+ when(sf.<Integration>create(eq("redshift"), eq("redshift-integrations.json"), any())).thenReturn(integrationBackend);
when(clusterBackend.accountId()).thenReturn("111111111111");
regionResolver = new RegionResolver("us-east-1", "111111111111");
credentialBroker = new RedshiftCredentialBroker();
secretsManagerService = mock(SecretsManagerService.class);
+ streamService = mock(DynamoDbStreamService.class);
service = new RedshiftService(sf, cm, config, regionResolver, proxyManager, dockerHostResolver,
- credentialBroker, secretsManagerService, new com.fasterxml.jackson.databind.ObjectMapper());
+ credentialBroker, secretsManagerService, new com.fasterxml.jackson.databind.ObjectMapper(),
+ streamService);
+ }
+
+ @Test
+ void createDynamoDbZeroEtlIntegrationRequiresExistingProvisionedResources() {
+ String streamArn = "arn:aws:dynamodb:us-east-1:111111111111:table/orders/stream/2026-09-18T00:00:00.000";
+ String targetArn = "arn:aws:redshift:us-east-1:111111111111:cluster:warehouse";
+ StreamDescription stream = new StreamDescription();
+ stream.setStreamArn(streamArn);
+ stream.setTableName("orders");
+ stream.setStreamStatus("ENABLED");
+ when(streamService.describeStream(streamArn)).thenReturn(stream);
+ Cluster cluster = new Cluster();
+ cluster.setClusterIdentifier("warehouse");
+ when(clusterBackend.get("warehouse")).thenReturn(Optional.of(cluster));
+ when(integrationBackend.scan(any())).thenReturn(List.of());
+
+ Integration integration = service.createIntegration("orders-to-warehouse", streamArn, targetArn,
+ null, null, Map.of(), Map.of(), "us-east-1");
+
+ assertEquals(streamArn, integration.getSourceStreamArn());
+ assertEquals("warehouse", integration.getTargetClusterIdentifier());
+ assertNotNull(integration.getLandingTableName());
+ }
+
+ @Test
+ void createDynamoDbZeroEtlIntegrationRejectsServerlessTarget() {
+ assertThrows(AwsException.class, () -> service.createIntegration(
+ "orders-to-serverless",
+ "arn:aws:dynamodb:us-east-1:111111111111:table/orders/stream/2026-09-18T00:00:00.000",
+ "arn:aws:redshift-serverless:us-east-1:111111111111:workgroup/analytics",
+ null, null, Map.of(), Map.of(), "us-east-1"));
}
/** Absolute dump path as {@code createSnapshot} now stores it: under {@code <persistentPath>/redshift-dumps/<accountId>}. */
@@ -390,6 +430,19 @@ class RedshiftServiceTest {
assertEquals("test-c", list.get(0).getClusterIdentifier());
}
+ @Test
+ void describeClustersForAccountUsesTheRequestedAccount() {
+ Cluster cluster = new Cluster();
+ cluster.setClusterIdentifier("test-c");
+ when(clusterBackend.getForAccount("222222222222", "test-c")).thenReturn(Optional.of(cluster));
+
+ List<Cluster> list = service.describeClustersForAccount("222222222222", "test-c");
+
+ assertEquals(List.of(cluster), list);
+ verify(clusterBackend).getForAccount("222222222222", "test-c");
+ verify(clusterBackend, never()).get("test-c");
+ }
+
@Test
void testDeleteCluster() {
Cluster c = new Cluster();
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriterTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriterTest.java
new file mode 100644
index 0000000000..33181964b5
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriterTest.java
@@ -0,0 +1,88 @@
+package io.github.hectorvent.floci.services.redshiftdata;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;
+import io.github.hectorvent.floci.services.redshift.RedshiftService;
+import io.github.hectorvent.floci.services.redshift.model.Cluster;
+import org.junit.jupiter.api.Test;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.Statement;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.contains;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.*;
+
+class RedshiftZeroEtlWriterTest {
+
+ @Test
+ void createsLandingTableWithStableEventColumns() throws Exception {
+ RedshiftService redshiftService = mock(RedshiftService.class);
+ RedshiftDataConnectionFactory connectionFactory = mock(RedshiftDataConnectionFactory.class);
+ ObjectMapper objectMapper = new ObjectMapper();
+ Connection connection = mock(Connection.class);
+ Statement statement = mock(Statement.class);
+ Cluster cluster = cluster();
+ when(redshiftService.describeClustersForAccount("111111111111", "warehouse")).thenReturn(List.of(cluster));
+ when(connectionFactory.open(any())).thenReturn(connection);
+ when(connection.createStatement()).thenReturn(statement);
+
+ RedshiftZeroEtlWriter writer = new RedshiftZeroEtlWriter(redshiftService, connectionFactory, objectMapper);
+ writer.createLandingTable("111111111111", "warehouse", "floci_zetl_orders");
+
+ verify(statement).executeUpdate(contains("event_id TEXT PRIMARY KEY"));
+ verify(statement).executeUpdate(contains("new_image_json TEXT"));
+ verify(connection).close();
+ }
+
+ @Test
+ void writesStreamRecordsIdempotentlyAndReturnsNewestSequence() throws Exception {
+ RedshiftService redshiftService = mock(RedshiftService.class);
+ RedshiftDataConnectionFactory connectionFactory = mock(RedshiftDataConnectionFactory.class);
+ ObjectMapper objectMapper = new ObjectMapper();
+ Connection connection = mock(Connection.class);
+ PreparedStatement statement = mock(PreparedStatement.class);
+ when(redshiftService.describeClustersForAccount("111111111111", "warehouse")).thenReturn(List.of(cluster()));
+ when(connectionFactory.open(any())).thenReturn(connection);
+ when(connection.prepareStatement(contains("ON CONFLICT (event_id) DO NOTHING"))).thenReturn(statement);
+
+ DynamoDbStreamRecord first = record("event-1", "000000000000000000001");
+ DynamoDbStreamRecord second = record("event-2", "000000000000000000002");
+ RedshiftZeroEtlWriter writer = new RedshiftZeroEtlWriter(redshiftService, connectionFactory, objectMapper);
+
+ String sequence = writer.writeBatch("111111111111", "warehouse", "floci_zetl_orders", List.of(first, second));
+
+ assertEquals("000000000000000000002", sequence);
+ verify(statement, times(2)).executeUpdate();
+ verify(statement).setString(1, "event-1");
+ verify(statement).setString(1, "event-2");
+ verify(connection).close();
+ }
+
+ private static Cluster cluster() {
+ Cluster cluster = new Cluster();
+ cluster.setClusterIdentifier("warehouse");
+ cluster.setContainerHost("localhost");
+ cluster.setContainerPort(5432);
+ cluster.setMasterUsername("admin");
+ cluster.setMasterPassword("password");
+ return cluster;
+ }
+
+ private static DynamoDbStreamRecord record(String eventId, String sequenceNumber) {
+ DynamoDbStreamRecord record = new DynamoDbStreamRecord();
+ record.setEventId(eventId);
+ record.setEventName("INSERT");
+ record.setEventSource("aws:dynamodb");
+ record.setAwsRegion("us-east-1");
+ record.setSequenceNumber(sequenceNumber);
+ record.setApproximateCreationDateTime(1_758_153_600L);
+ record.setKeys(JsonNodeFactory.instance.objectNode().put("id", "1"));
+ record.setNewImage(JsonNodeFactory.instance.objectNode().put("value", "new"));
+ return record;
+ }
+}
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 9a000e6712..df1e01a654 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -188,6 +188,7 @@ floci:
enabled: true
default-port: 5439
image-version: postgres:15-alpine
+ poll-interval-ms: 1000
proxy-base-port: 7100
proxy-max-port: 7199
default-credential-duration-seconds: 900