Light, fluffy, and always free - The AWS Local Emulator alternative
feat: add DynamoDB to Redshift zero-ETL replication (#3872)
Vũ Thế Huy29b601834b3d
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.mdindex 68754bdf14..a5c1c779a3 100644--- a/docs/services/dynamodb.md+++ b/docs/services/dynamodb.md@@ -50,6 +50,10 @@ DynamoDB Streams are supported via a separate target (`DynamoDBStreams_20120810`| `GetShardIterator` | Get a shard iterator || `GetRecords` | Read stream records from a shard |+Redshift zero-ETL integrations can consume these stream records directly. See the+[Redshift DynamoDB zero-ETL](redshift.md#dynamodb-zero-etl) section for the supported target,+landing table, checkpoint, and retry behavior.+## Configuration| Variable | Default | Description |diff --git a/docs/services/redshift.md b/docs/services/redshift.mdindex 5ee4cd5cc1..6dac6438dd 100644--- a/docs/services/redshift.md+++ b/docs/services/redshift.md@@ -33,7 +33,7 @@ For running SQL without a PostgreSQL wire connection (the way Lambda and Step Fu| `DeleteTags` | Remove tags by key from a resource || `DescribeTags` | List tagged resources and their tags || `CreateClusterSubnetGroup` | Register a cluster subnet group (metadata only) |-| `CreateIntegration` | Register a zero-ETL integration (metadata only; no data is replicated). Accepts `Description`, `KMSKeyId`, `AdditionalEncryptionContext` and `TagList` |+| `CreateIntegration` | Create a DynamoDB Streams to provisioned Redshift zero-ETL integration. Accepts `Description`, `KMSKeyId`, `AdditionalEncryptionContext` and `TagList` || `DescribeIntegrations` | List integrations with `Filters`, `MaxRecords` and `Marker` pagination, or the one an `IntegrationArn` names || `DeleteIntegration` | Remove a zero-ETL integration || `DescribeClusterSubnetGroups` | List subnet groups, optionally filtered by name |@@ -45,6 +45,21 @@ For running SQL without a PostgreSQL wire connection (the way Lambda and Step Fu| `GetClusterCredentialsWithIAM` | Issue short-lived credentials with the DbUser derived from the caller's IAM identity |<!-- floci:actions:end -->+## DynamoDB zero-ETL++The supported zero-ETL path is DynamoDB Streams to a provisioned Redshift cluster. The source ARN+must identify an enabled local DynamoDB stream, and the target ARN must identify an existing+provisioned Redshift cluster.++Each stream record is written to a stable landing table named `floci_zetl_<integration-id>`.+The landing table stores the event id, event name, source, region, sequence number, creation time,+and DynamoDB keys and images as JSON text. Record event ids are unique, so retries are idempotent.++The integration consumer has its own stream checkpoint and does not share Lambda event source+mapping state. Deleting an integration stops its consumer while retaining the landing table.+Serverless Redshift targets, backfill, schema inference, and relational projection are not supported+in this first implementation.+## CloudFormationFloci provisions these resource types:diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 8e67e37147..4d650105c0 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1396,6 +1396,9 @@ public interface EmulatorConfig {@WithDefault("7199")int proxyMaxPort();+ @WithDefault("1000")+ long pollIntervalMs();+// Hostname clients use to reach a cluster endpoint. Empty -> resolved from// DockerHostResolver (falls back to "localhost").Optional<String> endpointHost();diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.javanew file mode 100644index 0000000000..98d9bbd0ca--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumer.java@@ -0,0 +1,158 @@+package io.github.hectorvent.floci.services.redshift;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;+import io.github.hectorvent.floci.services.redshift.model.Integration;+import io.github.hectorvent.floci.services.redshiftdata.RedshiftZeroEtlWriter;+import io.quarkus.runtime.StartupEvent;+import io.vertx.core.Vertx;+import jakarta.annotation.PreDestroy;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;++@ApplicationScoped+public class RedshiftDynamoDbZeroEtlConsumer {++ private static final Logger LOG = Logger.getLogger(RedshiftDynamoDbZeroEtlConsumer.class);+ private static final int BATCH_SIZE = 100;++ private final Vertx vertx;+ private final DynamoDbStreamService streamService;+ private final RedshiftService redshiftService;+ private final RedshiftZeroEtlWriter writer;+ private final long pollIntervalMs;+ private final ConcurrentHashMap<String, Long> timerIds = new ConcurrentHashMap<>();+ private final ConcurrentHashMap<String, Boolean> activePolls = new ConcurrentHashMap<>();+ private final ExecutorService pollExecutor = Executors.newCachedThreadPool(runnable -> {+ Thread thread = new Thread(runnable, "redshift-zero-etl");+ thread.setDaemon(true);+ return thread;+ });++ @Inject+ public RedshiftDynamoDbZeroEtlConsumer(Vertx vertx,+ DynamoDbStreamService streamService,+ RedshiftService redshiftService,+ RedshiftZeroEtlWriter writer,+ EmulatorConfig config) {+ this(vertx, streamService, redshiftService, writer, config.services().redshift().pollIntervalMs());+ }++ RedshiftDynamoDbZeroEtlConsumer(DynamoDbStreamService streamService,+ RedshiftService redshiftService,+ RedshiftZeroEtlWriter writer) {+ this(null, streamService, redshiftService, writer, 1000);+ }++ private RedshiftDynamoDbZeroEtlConsumer(Vertx vertx,+ DynamoDbStreamService streamService,+ RedshiftService redshiftService,+ RedshiftZeroEtlWriter writer,+ long pollIntervalMs) {+ this.vertx = vertx;+ this.streamService = streamService;+ this.redshiftService = redshiftService;+ this.writer = writer;+ this.pollIntervalMs = pollIntervalMs;+ }++ void onStart(@Observes StartupEvent event) {+ startPersistedIntegrations();+ }++ public void startPersistedIntegrations() {+ for (Integration integration : redshiftService.listDynamoDbZeroEtlIntegrations()) {+ if (integration.isPollingEnabled()) {+ startPolling(integration);+ }+ }+ }++ public void startPolling(Integration integration) {+ if (vertx == null || timerIds.containsKey(integration.getIntegrationArn())) {+ return;+ }+ long timerId = vertx.setPeriodic(pollIntervalMs, ignored -> pollAnd(integration));+ timerIds.put(integration.getIntegrationArn(), timerId);+ }++ public void stopPolling(String integrationArn) {+ Long timerId = timerIds.remove(integrationArn);+ if (timerId != null) {+ vertx.cancelTimer(timerId);+ }+ activePolls.remove(integrationArn);+ }++ void pollOnce(Integration integration) {+ writer.createLandingTable(integration.getAccountId(), integration.getTargetClusterIdentifier(),+ integration.getLandingTableName());+ String iteratorType = integration.getCheckpointSequenceNumber() == null+ ? "TRIM_HORIZON" : "AFTER_SEQUENCE_NUMBER";+ String iterator = streamService.getShardIterator(integration.getSourceStreamArn(),+ DynamoDbStreamService.SHARD_ID, iteratorType, integration.getCheckpointSequenceNumber());+ DynamoDbStreamService.GetRecordsResult result = streamService.getRecords(iterator, BATCH_SIZE);+ if (result.records().isEmpty()) {+ return;+ }+ String sequence = writer.writeBatch(integration.getAccountId(), integration.getTargetClusterIdentifier(),+ integration.getLandingTableName(), result.records());+ redshiftService.updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),+ sequence, true, null);+ integration.setCheckpointSequenceNumber(sequence);+ }++ public void reset() {+ for (String integrationArn : timerIds.keySet()) {+ stopPolling(integrationArn);+ }+ activePolls.clear();+ }++ @PreDestroy+ void onStop() {+ reset();+ pollExecutor.shutdownNow();+ }++ private void pollAnd(Integration integration) {+ String integrationArn = integration.getIntegrationArn();+ if (activePolls.putIfAbsent(integrationArn, Boolean.TRUE) != null) {+ return;+ }+ pollExecutor.submit(() -> {+ try {+ pollSafely(integration);+ } finally {+ activePolls.remove(integrationArn);+ }+ });+ }++ private void pollSafely(Integration integration) {+ try {+ pollOnce(integration);+ } catch (Exception e) {+ LOG.warnv(e, "Zero-ETL polling failed for integration {0}", integration.getIntegrationArn());+ if (e instanceof AwsException awsException+ && "TrimmedDataAccessException".equals(awsException.getErrorCode())) {+ integration.setCheckpointSequenceNumber(null);+ }+ try {+ redshiftService.updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),+ integration.getCheckpointSequenceNumber(), false, e.getMessage());+ } catch (Exception updateError) {+ LOG.warnv(updateError, "Could not persist zero-ETL failure for integration {0}",+ integration.getIntegrationArn());+ }+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.javaindex 2ef5a99358..8dafed5def 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandler.java@@ -42,16 +42,25 @@ public class RedshiftQueryHandler {private final EmulatorConfig config;private final RedshiftIamDbUserResolver iamDbUserResolver;private final RegionResolver regionResolver;+ private final RedshiftDynamoDbZeroEtlConsumer zeroEtlConsumer;@Injectpublic RedshiftQueryHandler(RedshiftService service, RedshiftCredentialBroker credentialBroker,EmulatorConfig config, RedshiftIamDbUserResolver iamDbUserResolver,- RegionResolver regionResolver) {+ RegionResolver regionResolver,+ RedshiftDynamoDbZeroEtlConsumer zeroEtlConsumer) {this.service = service;this.credentialBroker = credentialBroker;this.config = config;this.iamDbUserResolver = iamDbUserResolver;this.regionResolver = regionResolver;+ this.zeroEtlConsumer = zeroEtlConsumer;+ }++ RedshiftQueryHandler(RedshiftService service, RedshiftCredentialBroker credentialBroker,+ EmulatorConfig config, RedshiftIamDbUserResolver iamDbUserResolver,+ RegionResolver regionResolver) {+ this(service, credentialBroker, config, iamDbUserResolver, regionResolver, null);}public Response handle(String action, MultivaluedMap<String, String> params) {@@ -372,6 +381,9 @@ public class RedshiftQueryHandler {encryptionContextMap(params),tagMap(params),regionResolver.resolveRegionFromAuth(authorizationHeader));+ if (zeroEtlConsumer != null) {+ zeroEtlConsumer.startPolling(integration);+ }String xml = new XmlBuilder().start("CreateIntegrationResponse").start("CreateIntegrationResult")@@ -414,6 +426,9 @@ public class RedshiftQueryHandler {}case "DeleteIntegration" -> {Integration integration = service.deleteIntegration(params.getFirst("IntegrationArn"));+ if (zeroEtlConsumer != null) {+ zeroEtlConsumer.stopPolling(integration.getIntegrationArn());+ }String xml = new XmlBuilder().start("DeleteIntegrationResponse").start("DeleteIntegrationResult")diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.javaindex 9dc645aa40..546ef96f14 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerHandle;import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerManager;@@ -74,6 +75,7 @@ public class RedshiftService {private final RedshiftCredentialBroker credentialBroker;private final SecretsManagerService secretsManagerService;private final ObjectMapper objectMapper;+ private final DynamoDbStreamService streamService;// Proxy ports currently handed out, so allocateProxyPort never double-assigns within this JVM.private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();@@ -82,7 +84,8 @@ public class RedshiftService {EmulatorConfig config, RegionResolver regionResolver,RedshiftProxyManager proxyManager, DockerHostResolver dockerHostResolver,RedshiftCredentialBroker credentialBroker,- SecretsManagerService secretsManagerService, ObjectMapper objectMapper) {+ SecretsManagerService secretsManagerService, ObjectMapper objectMapper,+ DynamoDbStreamService streamService) {this.clusters = storageFactory.create("redshift", "redshift-clusters.json", new TypeReference<Map<String, Cluster>>() {});this.snapshots = storageFactory.create("redshift", "redshift-snapshots.json", new TypeReference<Map<String, Snapshot>>() {});this.parameterGroups = storageFactory.create("redshift", "redshift-parameter-groups.json", new TypeReference<Map<String, ClusterParameterGroup>>() {});@@ -96,6 +99,7 @@ public class RedshiftService {this.credentialBroker = credentialBroker;this.secretsManagerService = secretsManagerService;this.objectMapper = objectMapper;+ this.streamService = streamService;}RedshiftService(StorageFactory storageFactory, RedshiftContainerManager containerManager,@@ -103,7 +107,7 @@ public class RedshiftService {RedshiftProxyManager proxyManager, DockerHostResolver dockerHostResolver,RedshiftCredentialBroker credentialBroker) {this(storageFactory, containerManager, config, regionResolver, proxyManager, dockerHostResolver,- credentialBroker, null, new ObjectMapper());+ credentialBroker, null, new ObjectMapper(), null);}// Recreate Docker containers for persisted clusters on app restart (across every account, not just default)@@ -314,6 +318,26 @@ public class RedshiftService {if (targetArn == null || targetArn.isBlank()) {throw new AwsException("InvalidParameterValue", "TargetArn is required.", 400);}+ AwsArnUtils.Arn source = parseZeroEtlArn(sourceArn, "DynamoDB stream");+ if (!"dynamodb".equals(source.service()) || !source.resource().startsWith("table/")+ || !source.resource().contains("/stream/")) {+ throw new AwsException("InvalidParameterValue",+ "SourceArn must identify a DynamoDB stream.", 400);+ }+ if (streamService == null) {+ throw new AwsException("InternalFailure", "DynamoDB stream service is unavailable.", 500);+ }+ streamService.describeStream(sourceArn);+ AwsArnUtils.Arn target = parseZeroEtlArn(targetArn, "Redshift cluster");+ if (!"redshift".equals(target.service()) || !target.resource().startsWith("cluster:")) {+ throw new AwsException("InvalidParameterValue",+ "TargetArn must identify a provisioned Redshift cluster.", 400);+ }+ String clusterIdentifier = target.resource().substring("cluster:".length());+ if (clusterIdentifier.isBlank()) {+ throw new AwsException("InvalidParameterValue", "TargetArn must identify a Redshift cluster.", 400);+ }+ describeClusters(clusterIdentifier);if (description != null && description.length() > MAX_INTEGRATION_DESCRIPTION) {throw new AwsException("InvalidParameterValue","Description must be at most " + MAX_INTEGRATION_DESCRIPTION + " characters.", 400);@@ -334,11 +358,19 @@ public class RedshiftService {String integrationId = UUID.randomUUID().toString();Integration integration = new Integration();+ integration.setAccountId(integrations.accountId());integration.setIntegrationArn("arn:aws:redshift:" + region + ":" + regionResolver.getAccountId()+ ":integration:" + integrationId);integration.setIntegrationName(integrationName);integration.setSourceArn(sourceArn);integration.setTargetArn(targetArn);+ integration.setSourceStreamArn(sourceArn);+ integration.setTargetClusterIdentifier(clusterIdentifier);+ integration.setLandingTableName("floci_zetl_" + integrationId.replace('-', '_'));+ integration.setCheckpointSequenceNumber(null);+ integration.setRetryCount(0);+ integration.setLastError(null);+ integration.setPollingEnabled(true);// Real integrations pass through creating before settling; nothing here has work to do.integration.setStatus("active");integration.setKmsKeyId(kmsKeyId);@@ -351,6 +383,14 @@ public class RedshiftService {return integration;}+ private static AwsArnUtils.Arn parseZeroEtlArn(String arn, String resourceType) {+ try {+ return AwsArnUtils.parse(arn);+ } catch (IllegalArgumentException e) {+ throw new AwsException("InvalidParameterValue", resourceType + " ARN is invalid.", 400);+ }+ }+/*** Lists integrations with the documented filters and marker pagination.*@@ -401,6 +441,45 @@ public class RedshiftService {return new IntegrationPage(List.copyOf(page), next);}+ public List<Integration> listDynamoDbZeroEtlIntegrations() {+ return integrations.scanAllAccountEntries(key -> true).stream()+ .filter(entry -> entry.value().getSourceStreamArn() != null+ && !entry.value().getSourceStreamArn().isBlank())+ .map(entry -> {+ Integration integration = entry.value();+ if (integration.getAccountId() == null) {+ integration.setAccountId(entry.accountId());+ }+ return integration;+ })+ .toList();+ }++ public synchronized void updateIntegrationRuntime(String accountId, String integrationArn,+ String checkpointSequenceNumber,+ boolean successful, String error) {+ for (String key : integrations.keysForAccount(accountId)) {+ Optional<Integration> stored = integrations.getForAccount(accountId, key);+ if (stored.isEmpty() || !integrationArn.equals(stored.get().getIntegrationArn())) {+ continue;+ }+ Integration integration = stored.get();+ if (successful) {+ integration.setCheckpointSequenceNumber(checkpointSequenceNumber);+ integration.setRetryCount(0);+ integration.setLastError(null);+ integration.setStatus("active");+ } else {+ integration.setRetryCount(integration.getRetryCount() + 1);+ integration.setLastError(error);+ integration.setStatus("failed");+ }+ integrations.putForAccount(accountId, key, integration);+ return;+ }+ throw new AwsException("IntegrationNotFoundFault", "The requested integration doesn't exist.", 404);+ }+/** One page of integrations plus the marker to continue with, or {@code null} at the end. */public record IntegrationPage(List<Integration> integrations, String marker) {}@@ -468,6 +547,17 @@ public class RedshiftService {return clusters.scan(k -> true);}+ public List<Cluster> describeClustersForAccount(String accountId, String identifier) {+ if (identifier != null) {+ Optional<Cluster> cluster = clusters.getForAccount(accountId, identifier);+ if (cluster.isEmpty()) {+ throw new AwsException("ClusterNotFound", "Cluster " + identifier + " not found", 404);+ }+ return List.of(cluster.get());+ }+ return clusters.scanForAccount(accountId, k -> true);+ }+public synchronized Cluster deleteCluster(String identifier) {Optional<Cluster> clusterOpt = clusters.get(identifier);if (clusterOpt.isEmpty()) {diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/model/Integration.java b/src/main/java/io/github/hectorvent/floci/services/redshift/model/Integration.javaindex 6edf08a41c..085b35b75e 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/model/Integration.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/model/Integration.java@@ -16,9 +16,17 @@ import java.util.Map;@RegisterForReflectionpublic class Integration {private String integrationArn;+ private String accountId;private String integrationName;private String sourceArn;private String targetArn;+ private String sourceStreamArn;+ private String targetClusterIdentifier;+ private String landingTableName;+ private String checkpointSequenceNumber;+ private int retryCount;+ private String lastError;+ private boolean pollingEnabled;private String description;private String status;private String kmsKeyId;@@ -31,12 +39,28 @@ public class Integration {public String getIntegrationArn() { return integrationArn; }public void setIntegrationArn(String integrationArn) { this.integrationArn = integrationArn; }+ public String getAccountId() { return accountId; }+ public void setAccountId(String accountId) { this.accountId = accountId; }public String getIntegrationName() { return integrationName; }public void setIntegrationName(String integrationName) { this.integrationName = integrationName; }public String getSourceArn() { return sourceArn; }public void setSourceArn(String sourceArn) { this.sourceArn = sourceArn; }public String getTargetArn() { return targetArn; }public void setTargetArn(String targetArn) { this.targetArn = targetArn; }+ public String getSourceStreamArn() { return sourceStreamArn; }+ public void setSourceStreamArn(String sourceStreamArn) { this.sourceStreamArn = sourceStreamArn; }+ public String getTargetClusterIdentifier() { return targetClusterIdentifier; }+ public void setTargetClusterIdentifier(String targetClusterIdentifier) { this.targetClusterIdentifier = targetClusterIdentifier; }+ public String getLandingTableName() { return landingTableName; }+ public void setLandingTableName(String landingTableName) { this.landingTableName = landingTableName; }+ public String getCheckpointSequenceNumber() { return checkpointSequenceNumber; }+ public void setCheckpointSequenceNumber(String checkpointSequenceNumber) { this.checkpointSequenceNumber = checkpointSequenceNumber; }+ public int getRetryCount() { return retryCount; }+ public void setRetryCount(int retryCount) { this.retryCount = retryCount; }+ public String getLastError() { return lastError; }+ public void setLastError(String lastError) { this.lastError = lastError; }+ public boolean isPollingEnabled() { return pollingEnabled; }+ public void setPollingEnabled(boolean pollingEnabled) { this.pollingEnabled = pollingEnabled; }public String getDescription() { return description; }public void setDescription(String description) { this.description = description; }public String getStatus() { return status; }diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriter.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriter.javanew file mode 100644index 0000000000..8d7cb06912--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriter.java@@ -0,0 +1,147 @@+package io.github.hectorvent.floci.services.redshiftdata;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;+import io.github.hectorvent.floci.services.redshift.RedshiftService;+import io.github.hectorvent.floci.services.redshift.model.Cluster;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;++import java.sql.Connection;+import java.sql.PreparedStatement;+import java.sql.SQLException;+import java.sql.Statement;+import java.sql.Types;+import java.time.Instant;+import java.util.List;+import java.util.regex.Pattern;++@ApplicationScoped+public class RedshiftZeroEtlWriter {++ private static final Pattern TABLE_NAME = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$");+ private static final String LANDING_TABLE_DDL = "CREATE TABLE IF NOT EXISTS %s ("+ + "event_id TEXT PRIMARY KEY, "+ + "event_name TEXT NOT NULL, "+ + "event_source TEXT, "+ + "aws_region TEXT, "+ + "sequence_number TEXT NOT NULL, "+ + "approximate_creation_datetime TIMESTAMPTZ, "+ + "keys_json TEXT, "+ + "old_image_json TEXT, "+ + "new_image_json TEXT, "+ + "ingested_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP";+ private static final String INSERT_RECORD = "INSERT INTO %s (event_id, event_name, event_source, aws_region, "+ + "sequence_number, approximate_creation_datetime, keys_json, old_image_json, new_image_json) "+ + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (event_id) DO NOTHING";++ private final RedshiftService redshiftService;+ private final RedshiftDataConnectionFactory connectionFactory;+ private final ObjectMapper objectMapper;++ @Inject+ RedshiftZeroEtlWriter(RedshiftService redshiftService,+ RedshiftDataConnectionFactory connectionFactory,+ ObjectMapper objectMapper) {+ this.redshiftService = redshiftService;+ this.connectionFactory = connectionFactory;+ this.objectMapper = objectMapper;+ }++ public void createLandingTable(String accountId, String clusterIdentifier, String tableName) {+ String safeTableName = validateTableName(tableName);+ try (Connection connection = connectionFactory.open(target(accountId, clusterIdentifier));+ Statement statement = connection.createStatement()) {+ statement.executeUpdate(LANDING_TABLE_DDL.formatted(safeTableName) + ")");+ } catch (SQLException e) {+ throw new AwsException("InternalFailure", "Could not create Redshift zero-ETL landing table.", 500);+ }+ }++ public String writeBatch(String accountId, String clusterIdentifier, String tableName,+ List<DynamoDbStreamRecord> records) {+ if (records == null || records.isEmpty()) {+ return null;+ }+ String safeTableName = validateTableName(tableName);+ String newestSequence = null;+ try (Connection connection = connectionFactory.open(target(accountId, clusterIdentifier));+ PreparedStatement statement = connection.prepareStatement(INSERT_RECORD.formatted(safeTableName))) {+ for (DynamoDbStreamRecord record : records) {+ bind(statement, record);+ statement.executeUpdate();+ newestSequence = record.getSequenceNumber();+ }+ } catch (SQLException e) {+ throw new AwsException("InternalFailure", "Could not write Redshift zero-ETL records.", 500);+ }+ return newestSequence;+ }++ private void bind(PreparedStatement statement, DynamoDbStreamRecord record) throws SQLException {+ statement.setString(1, record.getEventId());+ statement.setString(2, record.getEventName());+ setNullableString(statement, 3, record.getEventSource());+ setNullableString(statement, 4, record.getAwsRegion());+ statement.setString(5, record.getSequenceNumber());+ if (record.getApproximateCreationDateTime() > 0) {+ statement.setObject(6, Instant.ofEpochSecond(record.getApproximateCreationDateTime()));+ } else {+ statement.setNull(6, Types.TIMESTAMP_WITH_TIMEZONE);+ }+ setJson(statement, 7, record.getKeys());+ setJson(statement, 8, record.getOldImage());+ setJson(statement, 9, record.getNewImage());+ }++ private void setNullableString(PreparedStatement statement, int index, String value) throws SQLException {+ if (value == null) {+ statement.setNull(index, Types.VARCHAR);+ } else {+ statement.setString(index, value);+ }+ }++ private void setJson(PreparedStatement statement, int index, JsonNode value) throws SQLException {+ if (value == null || value.isNull()) {+ statement.setNull(index, Types.VARCHAR);+ return;+ }+ try {+ statement.setString(index, objectMapper.writeValueAsString(value));+ } catch (JsonProcessingException e) {+ throw new AwsException("InternalFailure", "Could not serialize a DynamoDB stream record.", 500);+ }+ }++ private RedshiftDataResourceResolver.DatabaseTarget target(String accountId, String clusterIdentifier) {+ Cluster cluster;+ try {+ cluster = redshiftService.describeClustersForAccount(accountId, clusterIdentifier).get(0);+ } catch (RuntimeException e) {+ throw new AwsException("ResourceNotFoundException",+ "Redshift cluster " + clusterIdentifier + " was not found.", 404);+ }+ if (cluster.getContainerHost() == null || cluster.getContainerHost().isBlank()+ || cluster.getContainerPort() <= 0) {+ throw new AwsException("InvalidClusterState", "Redshift cluster runtime is not available.", 400);+ }+ return new RedshiftDataResourceResolver.DatabaseTarget(+ "cluster:" + clusterIdentifier,+ cluster.getContainerHost(),+ cluster.getContainerPort(),+ "dev",+ cluster.getMasterUsername(),+ cluster.getMasterPassword());+ }++ private static String validateTableName(String tableName) {+ if (tableName == null || !TABLE_NAME.matcher(tableName).matches()) {+ throw new AwsException("InvalidParameterValue", "Landing table name is invalid.", 400);+ }+ return tableName;+ }+}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex ac4989b3cb..b988327aef 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -385,6 +385,7 @@ floci:enabled: truedefault-port: 5439image-version: postgres:15-alpine+ poll-interval-ms: 1000proxy-base-port: 7100proxy-max-port: 7199default-credential-duration-seconds: 900 # GetClusterCredentials default lifetime (900-3600)diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.javanew file mode 100644index 0000000000..e59b89545b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftDynamoDbZeroEtlConsumerTest.java@@ -0,0 +1,54 @@+package io.github.hectorvent.floci.services.redshift;++import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;+import io.github.hectorvent.floci.services.redshift.model.Integration;+import io.github.hectorvent.floci.services.redshiftdata.RedshiftZeroEtlWriter;+import org.junit.jupiter.api.Test;++import java.util.List;++import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.*;++class RedshiftDynamoDbZeroEtlConsumerTest {++ @Test+ void writesRecordsAndAdvancesCheckpointOnlyAfterSuccessfulBatch() {+ DynamoDbStreamService streamService = mock(DynamoDbStreamService.class);+ RedshiftService redshiftService = mock(RedshiftService.class);+ RedshiftZeroEtlWriter writer = mock(RedshiftZeroEtlWriter.class);+ Integration integration = integration();+ DynamoDbStreamRecord record = new DynamoDbStreamRecord();+ record.setEventId("event-1");+ record.setSequenceNumber("000000000000000000001");+ String iterator = "iterator";+ when(streamService.getShardIterator(eq(integration.getSourceStreamArn()), eq(DynamoDbStreamService.SHARD_ID),+ eq("TRIM_HORIZON"), eq(null))).thenReturn(iterator);+ when(streamService.getRecords(iterator, 100)).thenReturn(+ new DynamoDbStreamService.GetRecordsResult(List.of(record), "next"));+ when(writer.writeBatch(integration.getAccountId(), "warehouse", "floci_zetl_orders", List.of(record)))+ .thenReturn(record.getSequenceNumber());++ RedshiftDynamoDbZeroEtlConsumer consumer =+ new RedshiftDynamoDbZeroEtlConsumer(streamService, redshiftService, writer);+ consumer.pollOnce(integration);++ verify(writer).createLandingTable(integration.getAccountId(), "warehouse", "floci_zetl_orders");+ verify(writer).writeBatch(integration.getAccountId(), "warehouse", "floci_zetl_orders", List.of(record));+ verify(redshiftService).updateIntegrationRuntime(integration.getAccountId(), integration.getIntegrationArn(),+ record.getSequenceNumber(), true, null);+ }++ private static Integration integration() {+ Integration integration = new Integration();+ integration.setIntegrationArn("arn:aws:redshift:us-east-1:111111111111:integration:one");+ integration.setAccountId("111111111111");+ integration.setSourceStreamArn("arn:aws:dynamodb:us-east-1:111111111111:table/orders/stream/one");+ integration.setTargetClusterIdentifier("warehouse");+ integration.setLandingTableName("floci_zetl_orders");+ integration.setPollingEnabled(true);+ return integration;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftIntegrationsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftIntegrationsIntegrationTest.javaindex f8e58675c3..1845b4a979 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftIntegrationsIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftIntegrationsIntegrationTest.java@@ -2,9 +2,14 @@ package io.github.hectorvent.floci.services.redshift;import java.util.ArrayList;import java.util.List;+import io.restassured.RestAssured;+import io.restassured.response.Response;import io.quarkus.test.junit.QuarkusTest;+import io.restassured.config.HttpClientConfig;+import io.restassured.config.RestAssuredConfig;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.restassured.specification.RequestSpecification;-import io.restassured.response.Response;+import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import static io.restassured.RestAssured.given;@@ -26,9 +31,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals;@QuarkusTestclass RedshiftIntegrationsIntegrationTest {- private static final String SOURCE = "arn:aws:dynamodb:us-east-1:000000000000:table/keystone-main";- private static final String TARGET =- "arn:aws:redshift-serverless:us-east-1:000000000000:namespace/8445f0c7-d2b1-4c1c-916c-0eeaa68fd487";+ private static String source;+ private static String otherSource;+ private static final String TARGET = "arn:aws:redshift:us-east-1:000000000000:cluster:zero-etl-cluster";/*** The Authorization header is what routes a Query request to a service; without it the@@ -48,9 +53,50 @@ class RedshiftIntegrationsIntegrationTest {return spec.when().post("/");}+ @BeforeEach+ void createZeroEtlResources() {+ if (source != null) {+ return;+ }+ RestAssured.config = RestAssuredConfig.config()+ .httpClient(HttpClientConfig.httpClientConfig().setParam("http.socket.timeout", 180_000));+ RestAssuredJsonUtils.configureAwsContentTypes();+ String newSource = createDynamoTable("keystone-main");+ String newOtherSource = createDynamoTable("keystone-other");+ query("Action", "CreateCluster", "ClusterIdentifier", "zero-etl-cluster",+ "NodeType", "dc2.large", "MasterUsername", "admin", "MasterUserPassword", "password123")+ .then().statusCode(200);+ source = newSource;+ otherSource = newOtherSource;+ }++ private static String createDynamoTable(String tableName) {+ Response response = given()+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")+ .contentType("application/x-amz-json-1.0")+ .body("""+ {+ "TableName": "%s",+ "KeySchema": [{"AttributeName":"id","KeyType":"HASH"}],+ "AttributeDefinitions": [{"AttributeName":"id","AttributeType":"S"}],+ "BillingMode": "PAY_PER_REQUEST",+ "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}+ }+ """.formatted(tableName))+ .when().post("/");+ if (response.statusCode() != 200) {+ response = given()+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeTable")+ .contentType("application/x-amz-json-1.0")+ .body("{\"TableName\":\"%s\"}".formatted(tableName))+ .when().post("/");+ }+ return response.then().statusCode(200).extract().path("TableDescription.LatestStreamArn");+ }+private static String createIntegration(String name) {return query("Action", "CreateIntegration", "IntegrationName", name,- "SourceArn", SOURCE, "TargetArn", TARGET)+ "SourceArn", source, "TargetArn", TARGET).then().statusCode(200).extract().body().asString();}@@ -67,7 +113,7 @@ class RedshiftIntegrationsIntegrationTest {query("Action", "DescribeIntegrations").then().statusCode(200).body(containsString("zetl-described"))- .body(containsString(SOURCE))+ .body(containsString(source)).body(containsString(TARGET))// Lower case on real Redshift, not ACTIVE..body(containsString("<Status>active</Status>"))@@ -119,7 +165,7 @@ class RedshiftIntegrationsIntegrationTest {createIntegration("zetl-duplicate");query("Action", "CreateIntegration", "IntegrationName", "zetl-duplicate",- "SourceArn", SOURCE, "TargetArn", TARGET)+ "SourceArn", source, "TargetArn", TARGET).then().statusCode(400).body(containsString("IntegrationAlreadyExistsFault"));}@@ -132,7 +178,7 @@ class RedshiftIntegrationsIntegrationTest {// trailing hyphen are each outside it, so AWS refuses names floci used to accept.for (String name : new String[] {"1zetl", "-zetl", "zetl_name", "zetl--name", "zetl-"}) {query("Action", "CreateIntegration", "IntegrationName", name,- "SourceArn", SOURCE, "TargetArn", TARGET)+ "SourceArn", source, "TargetArn", TARGET).then().statusCode(400).body(containsString("InvalidParameterValue")).body(containsString("IntegrationName"));@@ -142,7 +188,7 @@ class RedshiftIntegrationsIntegrationTest {@Testvoid anIntegrationNameOverTheModelledLengthIsRejected() {query("Action", "CreateIntegration", "IntegrationName", "z".repeat(64),- "SourceArn", SOURCE, "TargetArn", TARGET)+ "SourceArn", source, "TargetArn", TARGET).then().statusCode(400).body(containsString("InvalidParameterValue"));}@@ -166,7 +212,7 @@ class RedshiftIntegrationsIntegrationTest {void tagsSurviveTheRoundTrip() {// The Query member is TagList, not Tags: an SDK serialises the list under its own name.query("Action", "CreateIntegration", "IntegrationName", "zetl-tagged",- "SourceArn", SOURCE, "TargetArn", TARGET,+ "SourceArn", source, "TargetArn", TARGET,"TagList.Tag.1.Key", "Environment", "TagList.Tag.1.Value", "dev").then().statusCode(200);@@ -187,7 +233,7 @@ class RedshiftIntegrationsIntegrationTest {@Testvoid descriptionAndEncryptionContextAreStoredAndReturned() {query("Action", "CreateIntegration", "IntegrationName", "zetl-full",- "SourceArn", SOURCE, "TargetArn", TARGET,+ "SourceArn", source, "TargetArn", TARGET,"Description", "nightly replica","KMSKeyId", "arn:aws:kms:us-east-1:000000000000:key/abc","AdditionalEncryptionContext.entry.1.key", "team",@@ -204,7 +250,7 @@ class RedshiftIntegrationsIntegrationTest {@Testvoid encryptionContextWithoutAKmsKeyIsRejected() {query("Action", "CreateIntegration", "IntegrationName", "zetl-nokms",- "SourceArn", SOURCE, "TargetArn", TARGET,+ "SourceArn", source, "TargetArn", TARGET,"AdditionalEncryptionContext.entry.1.key", "team","AdditionalEncryptionContext.entry.1.value", "data").then().statusCode(400)@@ -218,7 +264,7 @@ class RedshiftIntegrationsIntegrationTest {// rather than absolute counts.for (int i = 0; i < 21; i++) {query("Action", "CreateIntegration", "IntegrationName", String.format("zetl-page-%02d", i),- "SourceArn", SOURCE + "-" + i, "TargetArn", TARGET)+ "SourceArn", source, "TargetArn", TARGET).then().statusCode(200);}@@ -283,15 +329,15 @@ class RedshiftIntegrationsIntegrationTest {@Testvoid filteringBySourceArnNarrowsTheResult() {query("Action", "CreateIntegration", "IntegrationName", "zetl-filtered",- "SourceArn", SOURCE + "-filtered", "TargetArn", TARGET)+ "SourceArn", otherSource, "TargetArn", TARGET).then().statusCode(200);query("Action", "CreateIntegration", "IntegrationName", "zetl-unfiltered",- "SourceArn", SOURCE + "-other", "TargetArn", TARGET)+ "SourceArn", source, "TargetArn", TARGET).then().statusCode(200);query("Action", "DescribeIntegrations","Filters.DescribeIntegrationsFilter.1.Name", "source-arn",- "Filters.DescribeIntegrationsFilter.1.Values.Value.1", SOURCE + "-filtered")+ "Filters.DescribeIntegrationsFilter.1.Values.Value.1", otherSource).then().statusCode(200).body(containsString("zetl-filtered")).body(not(containsString("zetl-unfiltered")));diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.javaindex a428de1adf..871f629dcd 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java@@ -6,12 +6,15 @@ import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;+import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerHandle;import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerManager;import io.github.hectorvent.floci.services.redshift.model.Cluster;import io.github.hectorvent.floci.services.redshift.model.ClusterParameterGroup;import io.github.hectorvent.floci.services.redshift.model.ClusterSubnetGroup;import io.github.hectorvent.floci.services.redshift.model.Endpoint;+import io.github.hectorvent.floci.services.redshift.model.Integration;import io.github.hectorvent.floci.services.redshift.model.Parameter;import io.github.hectorvent.floci.services.redshift.model.Snapshot;import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;@@ -45,12 +48,14 @@ class RedshiftServiceTest {private AccountAwareStorageBackend<String> snapshotDumpBackend;private AccountAwareStorageBackend<ClusterParameterGroup> parameterGroupBackend;private AccountAwareStorageBackend<ClusterSubnetGroup> subnetGroupBackend;+ private AccountAwareStorageBackend<Integration> integrationBackend;private RedshiftContainerManager cm;private RegionResolver regionResolver;private RedshiftProxyManager proxyManager;private DockerHostResolver dockerHostResolver;private RedshiftCredentialBroker credentialBroker;private SecretsManagerService secretsManagerService;+ private DynamoDbStreamService streamService;private RedshiftService service;@BeforeEach@@ -62,6 +67,7 @@ class RedshiftServiceTest {snapshotDumpBackend = mock(AccountAwareStorageBackend.class);parameterGroupBackend = mock(AccountAwareStorageBackend.class);subnetGroupBackend = mock(AccountAwareStorageBackend.class);+ integrationBackend = mock(AccountAwareStorageBackend.class);cm = mock(RedshiftContainerManager.class);proxyManager = mock(RedshiftProxyManager.class);dockerHostResolver = mock(DockerHostResolver.class);@@ -86,15 +92,49 @@ class RedshiftServiceTest {when(sf.<Snapshot>create(eq("redshift"), eq("redshift-snapshots.json"), any())).thenReturn(snapshotBackend);when(sf.<ClusterParameterGroup>create(eq("redshift"), eq("redshift-parameter-groups.json"), any())).thenReturn(parameterGroupBackend);when(sf.<ClusterSubnetGroup>create(eq("redshift"), eq("redshift-subnet-groups.json"), any())).thenReturn(subnetGroupBackend);+ when(sf.<Integration>create(eq("redshift"), eq("redshift-integrations.json"), any())).thenReturn(integrationBackend);when(clusterBackend.accountId()).thenReturn("111111111111");regionResolver = new RegionResolver("us-east-1", "111111111111");credentialBroker = new RedshiftCredentialBroker();secretsManagerService = mock(SecretsManagerService.class);+ streamService = mock(DynamoDbStreamService.class);service = new RedshiftService(sf, cm, config, regionResolver, proxyManager, dockerHostResolver,- credentialBroker, secretsManagerService, new com.fasterxml.jackson.databind.ObjectMapper());+ credentialBroker, secretsManagerService, new com.fasterxml.jackson.databind.ObjectMapper(),+ streamService);+ }++ @Test+ void createDynamoDbZeroEtlIntegrationRequiresExistingProvisionedResources() {+ String streamArn = "arn:aws:dynamodb:us-east-1:111111111111:table/orders/stream/2026-09-18T00:00:00.000";+ String targetArn = "arn:aws:redshift:us-east-1:111111111111:cluster:warehouse";+ StreamDescription stream = new StreamDescription();+ stream.setStreamArn(streamArn);+ stream.setTableName("orders");+ stream.setStreamStatus("ENABLED");+ when(streamService.describeStream(streamArn)).thenReturn(stream);+ Cluster cluster = new Cluster();+ cluster.setClusterIdentifier("warehouse");+ when(clusterBackend.get("warehouse")).thenReturn(Optional.of(cluster));+ when(integrationBackend.scan(any())).thenReturn(List.of());++ Integration integration = service.createIntegration("orders-to-warehouse", streamArn, targetArn,+ null, null, Map.of(), Map.of(), "us-east-1");++ assertEquals(streamArn, integration.getSourceStreamArn());+ assertEquals("warehouse", integration.getTargetClusterIdentifier());+ assertNotNull(integration.getLandingTableName());+ }++ @Test+ void createDynamoDbZeroEtlIntegrationRejectsServerlessTarget() {+ assertThrows(AwsException.class, () -> service.createIntegration(+ "orders-to-serverless",+ "arn:aws:dynamodb:us-east-1:111111111111:table/orders/stream/2026-09-18T00:00:00.000",+ "arn:aws:redshift-serverless:us-east-1:111111111111:workgroup/analytics",+ null, null, Map.of(), Map.of(), "us-east-1"));}/** Absolute dump path as {@code createSnapshot} now stores it: under {@code <persistentPath>/redshift-dumps/<accountId>}. */@@ -390,6 +430,19 @@ class RedshiftServiceTest {assertEquals("test-c", list.get(0).getClusterIdentifier());}+ @Test+ void describeClustersForAccountUsesTheRequestedAccount() {+ Cluster cluster = new Cluster();+ cluster.setClusterIdentifier("test-c");+ when(clusterBackend.getForAccount("222222222222", "test-c")).thenReturn(Optional.of(cluster));++ List<Cluster> list = service.describeClustersForAccount("222222222222", "test-c");++ assertEquals(List.of(cluster), list);+ verify(clusterBackend).getForAccount("222222222222", "test-c");+ verify(clusterBackend, never()).get("test-c");+ }+@Testvoid testDeleteCluster() {Cluster c = new Cluster();diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriterTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriterTest.javanew file mode 100644index 0000000000..33181964b5--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftZeroEtlWriterTest.java@@ -0,0 +1,88 @@+package io.github.hectorvent.floci.services.redshiftdata;++import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.JsonNodeFactory;+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;+import io.github.hectorvent.floci.services.redshift.RedshiftService;+import io.github.hectorvent.floci.services.redshift.model.Cluster;+import org.junit.jupiter.api.Test;++import java.sql.Connection;+import java.sql.PreparedStatement;+import java.sql.Statement;+import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.mockito.ArgumentMatchers.contains;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.*;++class RedshiftZeroEtlWriterTest {++ @Test+ void createsLandingTableWithStableEventColumns() throws Exception {+ RedshiftService redshiftService = mock(RedshiftService.class);+ RedshiftDataConnectionFactory connectionFactory = mock(RedshiftDataConnectionFactory.class);+ ObjectMapper objectMapper = new ObjectMapper();+ Connection connection = mock(Connection.class);+ Statement statement = mock(Statement.class);+ Cluster cluster = cluster();+ when(redshiftService.describeClustersForAccount("111111111111", "warehouse")).thenReturn(List.of(cluster));+ when(connectionFactory.open(any())).thenReturn(connection);+ when(connection.createStatement()).thenReturn(statement);++ RedshiftZeroEtlWriter writer = new RedshiftZeroEtlWriter(redshiftService, connectionFactory, objectMapper);+ writer.createLandingTable("111111111111", "warehouse", "floci_zetl_orders");++ verify(statement).executeUpdate(contains("event_id TEXT PRIMARY KEY"));+ verify(statement).executeUpdate(contains("new_image_json TEXT"));+ verify(connection).close();+ }++ @Test+ void writesStreamRecordsIdempotentlyAndReturnsNewestSequence() throws Exception {+ RedshiftService redshiftService = mock(RedshiftService.class);+ RedshiftDataConnectionFactory connectionFactory = mock(RedshiftDataConnectionFactory.class);+ ObjectMapper objectMapper = new ObjectMapper();+ Connection connection = mock(Connection.class);+ PreparedStatement statement = mock(PreparedStatement.class);+ when(redshiftService.describeClustersForAccount("111111111111", "warehouse")).thenReturn(List.of(cluster()));+ when(connectionFactory.open(any())).thenReturn(connection);+ when(connection.prepareStatement(contains("ON CONFLICT (event_id) DO NOTHING"))).thenReturn(statement);++ DynamoDbStreamRecord first = record("event-1", "000000000000000000001");+ DynamoDbStreamRecord second = record("event-2", "000000000000000000002");+ RedshiftZeroEtlWriter writer = new RedshiftZeroEtlWriter(redshiftService, connectionFactory, objectMapper);++ String sequence = writer.writeBatch("111111111111", "warehouse", "floci_zetl_orders", List.of(first, second));++ assertEquals("000000000000000000002", sequence);+ verify(statement, times(2)).executeUpdate();+ verify(statement).setString(1, "event-1");+ verify(statement).setString(1, "event-2");+ verify(connection).close();+ }++ private static Cluster cluster() {+ Cluster cluster = new Cluster();+ cluster.setClusterIdentifier("warehouse");+ cluster.setContainerHost("localhost");+ cluster.setContainerPort(5432);+ cluster.setMasterUsername("admin");+ cluster.setMasterPassword("password");+ return cluster;+ }++ private static DynamoDbStreamRecord record(String eventId, String sequenceNumber) {+ DynamoDbStreamRecord record = new DynamoDbStreamRecord();+ record.setEventId(eventId);+ record.setEventName("INSERT");+ record.setEventSource("aws:dynamodb");+ record.setAwsRegion("us-east-1");+ record.setSequenceNumber(sequenceNumber);+ record.setApproximateCreationDateTime(1_758_153_600L);+ record.setKeys(JsonNodeFactory.instance.objectNode().put("id", "1"));+ record.setNewImage(JsonNodeFactory.instance.objectNode().put("value", "new"));+ return record;+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 9a000e6712..df1e01a654 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -188,6 +188,7 @@ floci:enabled: truedefault-port: 5439image-version: postgres:15-alpine+ poll-interval-ms: 1000proxy-base-port: 7100proxy-max-port: 7199default-credential-duration-seconds: 900