Light, fluffy, and always free - The AWS Local Emulator alternative
feat(firehose): deliver staged Redshift batches with single-attempt COPY (#6006)
Vũ Thế Huyfd37e109f8c8
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseRedshiftTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseRedshiftTest.javanew file mode 100644index 0000000000..be519f5b29--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseRedshiftTest.java@@ -0,0 +1,183 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.core.SdkBytes;+import software.amazon.awssdk.services.firehose.FirehoseClient;+import software.amazon.awssdk.services.firehose.model.BufferingHints;+import software.amazon.awssdk.services.firehose.model.CopyCommand;+import software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamRequest;+import software.amazon.awssdk.services.firehose.model.DeleteDeliveryStreamRequest;+import software.amazon.awssdk.services.firehose.model.DescribeDeliveryStreamRequest;+import software.amazon.awssdk.services.firehose.model.DestinationDescription;+import software.amazon.awssdk.services.firehose.model.PutRecordRequest;+import software.amazon.awssdk.services.firehose.model.Record;+import software.amazon.awssdk.services.firehose.model.RedshiftDestinationConfiguration;+import software.amazon.awssdk.services.firehose.model.ResourceNotFoundException;+import software.amazon.awssdk.services.firehose.model.S3DestinationConfiguration;+import software.amazon.awssdk.services.redshift.RedshiftClient;+import software.amazon.awssdk.services.redshift.model.ClusterNotFoundException;+import software.amazon.awssdk.services.redshift.model.CreateClusterRequest;+import software.amazon.awssdk.services.redshift.model.DeleteClusterRequest;+import software.amazon.awssdk.services.redshiftdata.RedshiftDataClient;+import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementRequest;+import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementResponse;+import software.amazon.awssdk.services.redshiftdata.model.ExecuteStatementRequest;+import software.amazon.awssdk.services.redshiftdata.model.GetStatementResultRequest;+import software.amazon.awssdk.services.redshiftdata.model.StatusString;++import java.nio.charset.StandardCharsets;+import java.time.Duration;+import java.time.Instant;+import java.util.UUID;++import static org.assertj.core.api.Assertions.assertThat;++@DisplayName("Firehose Redshift destination")+class FirehoseRedshiftTest {++ private static final String STREAM = "sdk-rs-shape-" + UUID.randomUUID().toString().substring(0, 8);+ private static final String JDBC_URL = "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev";+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose";+ private static FirehoseClient firehose;++ @BeforeAll+ static void setUp() {+ firehose = TestFixtures.firehoseClient();+ }++ @AfterAll+ static void tearDown() {+ if (firehose != null) {+ try {+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder().deliveryStreamName(STREAM).build());+ } catch (ResourceNotFoundException ignored) {+ // The stream does not exist when a test failed before creating it.+ }+ firehose.close();+ }+ }++ @Test+ @DisplayName("CreateDeliveryStream stores the Redshift destination and DescribeDeliveryStream returns it")+ void createAndDescribe() {+ firehose.createDeliveryStream(CreateDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM)+ .redshiftDestinationConfiguration(RedshiftDestinationConfiguration.builder()+ .roleARN(ROLE_ARN)+ .clusterJDBCURL(JDBC_URL)+ .username("admin")+ .password("Secret123")+ .copyCommand(CopyCommand.builder().dataTableName("events").copyOptions("json 'auto'").build())+ .s3Configuration(S3DestinationConfiguration.builder()+ .roleARN(ROLE_ARN)+ .bucketARN("arn:aws:s3:::sdk-rs-stage")+ .build())+ .build())+ .build());++ DestinationDescription destination = firehose.describeDeliveryStream(+ DescribeDeliveryStreamRequest.builder().deliveryStreamName(STREAM).build())+ .deliveryStreamDescription().destinations().get(0);++ assertThat(destination.redshiftDestinationDescription().clusterJDBCURL()).isEqualTo(JDBC_URL);+ assertThat(destination.redshiftDestinationDescription().username()).isEqualTo("admin");+ assertThat(destination.redshiftDestinationDescription().copyCommand().dataTableName()).isEqualTo("events");+ assertThat(destination.redshiftDestinationDescription().s3DestinationDescription().bucketARN())+ .isEqualTo("arn:aws:s3:::sdk-rs-stage");+ }++ @Test+ @DisplayName("PutRecord is loaded into the Redshift table through COPY")+ void deliversIntoTheRedshiftTable() throws Exception {+ String clusterId = TestFixtures.uniqueName("fh-rs-cluster");+ String stream = TestFixtures.uniqueName("fh-rs-delivery");+ String table = "fh_events";+ RedshiftClient redshift = TestFixtures.redshiftClient();+ RedshiftDataClient data = TestFixtures.redshiftDataClient();+ try {+ redshift.createCluster(CreateClusterRequest.builder()+ .clusterIdentifier(clusterId)+ .nodeType("dc2.large")+ .masterUsername("admin")+ .masterUserPassword("password123")+ .build());+ awaitFinished(data, execute(data, clusterId, "CREATE TABLE " + table + " (id int, name varchar(20))"));++ firehose.createDeliveryStream(CreateDeliveryStreamRequest.builder()+ .deliveryStreamName(stream)+ .redshiftDestinationConfiguration(RedshiftDestinationConfiguration.builder()+ .roleARN(ROLE_ARN)+ .clusterJDBCURL("jdbc:redshift://" + clusterId+ + ".abc123.us-east-1.redshift.amazonaws.com:5439/dev")+ .username("admin")+ .password("password123")+ .copyCommand(CopyCommand.builder()+ .dataTableName(table).copyOptions("json 'auto'").build())+ .s3Configuration(S3DestinationConfiguration.builder()+ .roleARN(ROLE_ARN)+ .bucketARN("arn:aws:s3:::" + stream + "-stage")+ .bufferingHints(BufferingHints.builder()+ .intervalInSeconds(60).sizeInMBs(1).build())+ .build())+ .build())+ .build());+ firehose.putRecord(PutRecordRequest.builder()+ .deliveryStreamName(stream)+ .record(Record.builder().data(SdkBytes.fromString("{\"id\":1,\"name\":\"alice\"}",+ StandardCharsets.UTF_8)).build())+ .build());++ Instant deadline = Instant.now().plus(Duration.ofSeconds(150));+ long rows = 0;+ while (rows == 0 && Instant.now().isBefore(deadline)) {+ String id = awaitFinished(data, execute(data, clusterId, "SELECT count(*) FROM " + table));+ rows = data.getStatementResult(GetStatementResultRequest.builder().id(id).build())+ .records().get(0).get(0).longValue();+ if (rows == 0) {+ Thread.sleep(5000);+ }+ }+ assertThat(rows).isEqualTo(1L);+ } finally {+ try {+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder().deliveryStreamName(stream).build());+ } catch (ResourceNotFoundException ignored) {+ // The stream does not exist when creation failed.+ }+ try {+ redshift.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(clusterId).build());+ } catch (ClusterNotFoundException ignored) {+ // The cluster does not exist when creation failed.+ }+ data.close();+ redshift.close();+ }+ }++ private static String execute(RedshiftDataClient data, String clusterId, String sql) {+ return data.executeStatement(ExecuteStatementRequest.builder()+ .clusterIdentifier(clusterId)+ .dbUser("admin")+ .database("dev")+ .sql(sql)+ .build()).id();+ }++ private static String awaitFinished(RedshiftDataClient data, String id) throws InterruptedException {+ Instant deadline = Instant.now().plus(Duration.ofSeconds(30));+ while (Instant.now().isBefore(deadline)) {+ DescribeStatementResponse describe = data.describeStatement(DescribeStatementRequest.builder().id(id).build());+ if (describe.status() == StatusString.FINISHED) {+ return id;+ }+ if (describe.status() == StatusString.FAILED || describe.status() == StatusString.ABORTED) {+ throw new IllegalStateException("Statement " + id + " ended " + describe.status() + ": " + describe.error());+ }+ Thread.sleep(250);+ }+ throw new IllegalStateException("Statement " + id + " did not finish in time");+ }+}diff --git a/docs/services/firehose.md b/docs/services/firehose.mdindex ca8a01a8fb..0bf758ef3d 100644--- a/docs/services/firehose.md+++ b/docs/services/firehose.md@@ -142,17 +142,52 @@ quoted value a `;`, a `$` (dollar quoting is never needed) and the comment markeunterminated quote. One closing `;` at the very end is the exception (the API reference's own example,`delimiter '\t' lzop;`, is accepted). Create and update reject anything else.-This release accepts and stores the destination and stages each batch in the intermediate S3 location, like an-S3 destination. It does not run the `COPY` into the cluster yet, and `S3BackupMode` has no effect on what is written.+### Delivery++A flush writes the batch to the intermediate S3 location, then runs+`COPY <DataTableName> [(<DataTableColumns>)] FROM 's3://<bucket>/<key>' <CopyOptions>` against the cluster. A single+closing `;` in `CopyOptions` is dropped from the statement. The `COPY` names no `IAM_ROLE`: AWS gives the cluster+temporary credentials of the stream's `RoleARN`, and Floci instead lets the proxy read the staged object from its own+S3, so `RoleARN` is validated and stored but not used for the load (an `IAM_ROLE` would have to exist in IAM and be+associated with the cluster). The `COPY` goes through the cluster's wire proxy, so it behaves like a `COPY` sent by any+other client (see [Redshift COPY from S3](redshift.md#copy-from-s3)). Credentials come from `Username` and `Password`,+or from the secret in `SecretsManagerConfiguration` when `Enabled` is true (the secret JSON needs `username` and+`password`). An empty `DataTableColumns` omits the column list, so the proxy discovers the table columns.++`DataTableName` and `DataTableColumns` are passed to the `COPY` as written, so Redshift's identifier rules apply+(an unquoted name is lowercased, a quoted one is kept).++A batch is staged in S3 once, and the `COPY` runs on its own thread so that a slow or hanging query does not hold the+buffer flusher thread or `PutRecord`. If its `COPY` fails (missing table or cluster, bad data), the failure is logged+and the staged object stays in S3; retrying failed loads will follow in a separate change. With `S3BackupMode` set to+`Enabled`, which needs an `S3BackupConfiguration`, each batch is also written once to that bucket under its own prefix+and compression. The backup is written after the batch is staged, and a failed backup write is logged without holding+the load back or re-staging the batch, so that batch then has no backup copy.++A JDBC connection that stops answering is dropped after 150 seconds.++Compression of the intermediate objects is not added to the `COPY` for you: as on AWS, put `GZIP` in `CopyOptions`+when the intermediate `CompressionFormat` is `GZIP`.++Floci does not write the `redshift_failed` manifest objects that AWS creates after a failed `COPY`.### Known deviations from AWS- `ClusterJDBCURL` must match the AWS pattern, so it needs a host such as`mycluster.abc123.us-east-1.redshift.amazonaws.com`. Floci treats the URL as a logical address: the firsthost label is the cluster identifier, and the host suffix and port are ignored. The database name is ignored- too, because the emulated database is always `dev`. The cluster is not looked up when the stream is created, so a stream- can be created before its cluster.+ too, because the emulated database is always `dev`. The cluster is looked up at delivery time, so a stream can+ be created before its cluster.- `ProcessingConfiguration` on a Redshift destination is accepted and returned but not executed.+- A `ClusterJDBCURL` with a `redshift-serverless` host is accepted, but delivery looks the name up as a provisioned+ cluster, so a serverless workgroup is not a delivery target yet.+- The `COPY` carries no credentials, so the proxy reads the staged object as an anonymous client. With S3+ authorization enforced and a private intermediate bucket the load fails and the staged object stays in S3; use a+ bucket the anonymous principal can read, or leave enforcement off.+- `CopyOptions` are validated for shape only. The `COPY` emulation does not implement every Redshift option+ (`LZOP`, `BZIP2`, `ZSTD`, `PARQUET` and others): such a statement is forwarded to PostgreSQL and fails, and the staged+ object stays in S3; retrying failed loads will follow in a separate change. Firehose never writes an `LZOP` object,+ so the API reference's `lzop` example would fail on AWS too.- The intermediate `Prefix` must not contain whitespace or quotes, because the `COPY` emulation cannot read suchobject keys. `DataFormatConversionConfiguration` is ignored on the Redshift `S3Configuration`, which AWS does notdefine.diff --git a/docs/services/redshift.md b/docs/services/redshift.mdindex c1b2020a99..61c0e84c3c 100644--- a/docs/services/redshift.md+++ b/docs/services/redshift.md@@ -442,6 +442,9 @@ order) through its own S3 service and streams the rows into the backing PostgreSbind parameters. Zero-parameter JDBC `PreparedStatement` calls therefore work with pgjdbc'sdefault extended mode. Statements containing bind parameters are forwarded unchanged.+Firehose delivery streams with a Redshift destination use this same path: Firehose stages each batch in S3 and+runs the `COPY` over a connection to the cluster proxy. See [Firehose](firehose.md#redshift-destination).+### Limitations- DDL rewriting works in both Simple Query (`'Q'`) and Extended Query (`Parse`) flows. COPY anddiff --git a/src/main/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentials.java b/src/main/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentials.javanew file mode 100644index 0000000000..3d75d30c8c--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentials.java@@ -0,0 +1,70 @@+package io.github.hectorvent.floci.core.common;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import org.jboss.logging.Logger;++import java.util.Objects;+import java.util.Optional;++/**+ * Username and password credentials stored in an AWS Secrets Manager secret for a database (RDS,+ * Redshift, Redshift Data API, or Firehose Redshift destination).+ */+public record DatabaseSecretCredentials(String username, String password) {++ private static final Logger LOG = Logger.getLogger(DatabaseSecretCredentials.class);+ private static final ObjectMapper DEFAULT_MAPPER = new ObjectMapper();++ public DatabaseSecretCredentials {+ Objects.requireNonNull(username, "username must not be null");+ Objects.requireNonNull(password, "password must not be null");+ }++ /**+ * Parses credentials from a Secrets Manager secret string JSON payload using the default mapper.+ */+ public static Optional<DatabaseSecretCredentials> parse(String secretString) {+ return parse(DEFAULT_MAPPER, secretString);+ }++ /**+ * Parses credentials from a Secrets Manager secret string using the given ObjectMapper.+ */+ public static Optional<DatabaseSecretCredentials> parse(ObjectMapper mapper, String secretString) {+ if (secretString == null || secretString.isBlank() || mapper == null) {+ return Optional.empty();+ }+ try {+ return parse(mapper.readTree(secretString));+ } catch (Exception ignored) {+ // An unparseable secret string is treated as missing credentials; callers handle Optional.empty()+ LOG.debugv("Failed to parse database credentials from secret string: {0}", ignored.getMessage());+ return Optional.empty();+ }+ }++ /**+ * Parses credentials from a Secrets Manager secret JSON node.+ * Looks for non-null "username" (or fallback "user") and "password" text fields.+ */+ public static Optional<DatabaseSecretCredentials> parse(JsonNode node) {+ if (node == null || !node.isObject()) {+ return Optional.empty();+ }+ String username = textOrNull(node, "username");+ if (username == null) {+ username = textOrNull(node, "user");+ }+ String password = textOrNull(node, "password");+ if (username != null && password != null) {+ return Optional.of(new DatabaseSecretCredentials(username, password));+ }+ return Optional.empty();+ }++ private static String textOrNull(JsonNode node, String field) {+ JsonNode child = node.get(field);+ return (child != null && !child.isNull()) ? child.asText() : null;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDelivery.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDelivery.javanew file mode 100644index 0000000000..9fd7f06621--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDelivery.java@@ -0,0 +1,150 @@+package io.github.hectorvent.floci.services.firehose;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.DatabaseSecretCredentials;+import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;+import io.github.hectorvent.floci.services.redshift.RedshiftService;+import io.github.hectorvent.floci.services.redshift.model.Cluster;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;+import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.sql.Connection;+import java.sql.DriverManager;+import java.sql.SQLException;+import java.sql.Statement;+import java.util.Properties;++/**+ * Loads a delivered S3 object into a Floci Redshift cluster. The {@code COPY} runs over a JDBC+ * connection to the cluster's wire proxy in simple query mode, so the proxy's S3 COPY emulation+ * does the work exactly as it does for any other client.+ */+@ApplicationScoped+public class FirehoseRedshiftDelivery {++ private static final Logger LOG = Logger.getLogger(FirehoseRedshiftDelivery.class);+ private static final int CONNECT_TIMEOUT_SECONDS = 10;+ private static final int SOCKET_TIMEOUT_SECONDS = 150;++ /** Seam for tests; production opens a real JDBC connection. */+ interface SqlRunner {+ void execute(String jdbcUrl, String user, String password, String sql) throws SQLException;+ }++ private final RedshiftService redshiftService;+ private final SecretsManagerService secretsManagerService;+ private final ObjectMapper mapper;+ private final SqlRunner runner;++ @Inject+ public FirehoseRedshiftDelivery(RedshiftService redshiftService, SecretsManagerService secretsManagerService,+ ObjectMapper mapper) {+ this(redshiftService, secretsManagerService, mapper, FirehoseRedshiftDelivery::runJdbc);+ }++ FirehoseRedshiftDelivery(RedshiftService redshiftService, SecretsManagerService secretsManagerService,+ ObjectMapper mapper, SqlRunner runner) {+ this.redshiftService = redshiftService;+ this.secretsManagerService = secretsManagerService;+ this.mapper = mapper;+ this.runner = runner;+ }++ /**+ * Runs the destination's COPY for the object at {@code s3://bucket/key}.+ *+ * Throws {@code AwsException} when the cluster or the secret cannot be resolved.+ *+ * @throws IllegalStateException when the statement itself fails; the message carries the table and SQL state+ */+ public void copy(String accountId, String region, RedshiftDestination destination, String bucket, String key) {+ RedshiftJdbcUrl target = RedshiftJdbcUrl.parse(destination.getClusterJdbcUrl());+ Cluster cluster = redshiftService.describeClustersForAccount(accountId, target.clusterIdentifier()).get(0);+ String[] credentials = credentials(destination, cluster, region);+ RedshiftDestination.CopyCommand command = destination.getCopyCommand();+ String sql = RedshiftCopySql.build(command.getDataTableName(), command.getDataTableColumns(), bucket, key,+ command.getCopyOptions());+ // The database name in the URL is ignored: the emulated container database is always "dev".+ String jdbcUrl = "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort()+ + "/dev?preferQueryMode=simple";+ try {+ runner.execute(jdbcUrl, credentials[0], credentials[1], sql);+ } catch (SQLException e) {+ throw new IllegalStateException("COPY into " + command.getDataTableName() + " on cluster "+ + target.clusterIdentifier() + " failed (SQLSTATE " + e.getSQLState() + "): " + e.getMessage(), e);+ }+ LOG.debugv("Firehose copied s3://{0}/{1} into {2} on cluster {3}", bucket, key,+ command.getDataTableName(), target.clusterIdentifier());+ }++ private String[] credentials(RedshiftDestination destination, Cluster cluster, String region) {+ RedshiftDestination.SecretsManagerConfiguration secrets = destination.getSecretsManagerConfiguration();+ if (secrets != null && Boolean.TRUE.equals(secrets.getEnabled())) {+ SecretVersion secret = secretsManagerService.getSecretValue(secrets.getSecretArn(), null, null, region);+ DatabaseSecretCredentials credentials = DatabaseSecretCredentials.parse(mapper, secret.getSecretString())+ .orElseThrow(() -> new IllegalStateException("Secret " + secrets.getSecretArn()+ + " does not contain username and password fields."));+ return new String[] {credentials.username(), credentials.password()};+ }+ if (destination.getPassword() == null) {+ throw new IllegalStateException("The Redshift destination for cluster " + cluster.getClusterIdentifier()+ + " has no Password and no enabled SecretsManagerConfiguration, so Firehose cannot sign in.");+ }+ String user = destination.getUsername() != null ? destination.getUsername() : cluster.getMasterUsername();+ return new String[] {user, destination.getPassword()};+ }++ /**+ * Timeouts matter here: a connection that never answers would stall the Redshift delivery thread+ * and delay emulator shutdown.+ */+ static Properties connectionProperties(String user, String password) {+ Properties properties = new Properties();+ properties.setProperty("user", user);+ properties.setProperty("password", password);+ properties.setProperty("connectTimeout", String.valueOf(CONNECT_TIMEOUT_SECONDS));+ properties.setProperty("socketTimeout", String.valueOf(SOCKET_TIMEOUT_SECONDS));+ return properties;+ }++ private static void runJdbc(String jdbcUrl, String user, String password, String sql) throws SQLException {+ Connection connection = DriverManager.getConnection(jdbcUrl, connectionProperties(user, password));+ executeAndClose(connection, sql);+ }++ /**+ * Runs the statement and closes the connection. Once the statement has returned the load is committed+ * (autocommit), so a failure while closing must not surface as a failed delivery.+ */+ static void executeAndClose(Connection connection, String sql) throws SQLException {+ try {+ // Closing the connection closes its statement, so the statement is not closed on its own: a failure there+ // would be reported as a failed COPY too.+ Statement statement = connection.createStatement();+ statement.execute(sql);+ } catch (SQLException e) {+ closeAfterFailure(connection, e);+ throw e;+ }+ closeAfterSuccess(connection);+ }++ private static void closeAfterFailure(Connection connection, SQLException failure) {+ try {+ connection.close();+ } catch (SQLException e) {+ failure.addSuppressed(e);+ }+ }++ private static void closeAfterSuccess(Connection connection) {+ try {+ connection.close();+ } catch (SQLException e) {+ LOG.warnv(e, "Closing the Redshift connection after a committed COPY failed: {0}", e.getMessage());+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 6959acdf75..9f85cb7e70 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -30,16 +30,21 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.io.ByteArrayOutputStream;+import java.io.IOException;import java.time.Clock;import java.time.Instant;-import java.util.*;import java.util.ArrayList;+import java.util.Collections;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import java.util.Optional;import java.util.Set;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.Executor;+import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;+import java.util.concurrent.RejectedExecutionException;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.TimeUnit;@@ -112,6 +117,8 @@ public class FirehoseService implements ResourceProvider {private final Map<String, Map<String, String>> pendingSourceIterators = new ConcurrentHashMap<>();private final FirehoseParquetConverter parquetConverter;private final FirehoseLambdaTransformer lambdaTransformer;+ private final FirehoseRedshiftDelivery redshiftDelivery;+ private final Executor redshiftDeliveryExecutor;// Serialises the read, version check, merge and write of a Redshift UpdateDestination per stream.private final Map<String, Object> redshiftUpdateLocks = new ConcurrentHashMap<>();private final Clock clock;@@ -196,11 +203,39 @@ public class FirehoseService implements ResourceProvider {sourceIteratorStore.deleteForAccount(accountFromScopedKey(key), logicalKeyFromScopedKey(key));}- @Inject+ /** Kept so callers that never deliver to Redshift need no delivery collaborator; null means unavailable. */public FirehoseService(StorageFactory storageFactory, S3Service s3Service, KinesisService kinesisService,RegionResolver regionResolver, Clock clock, EmulatorConfig config,FirehoseParquetConverter parquetConverter,FirehoseLambdaTransformer lambdaTransformer) {+ this(storageFactory, s3Service, kinesisService, regionResolver, clock, config, parquetConverter,+ lambdaTransformer, null);+ }++ @Inject+ public FirehoseService(StorageFactory storageFactory, S3Service s3Service, KinesisService kinesisService,+ RegionResolver regionResolver, Clock clock, EmulatorConfig config,+ FirehoseParquetConverter parquetConverter,+ FirehoseLambdaTransformer lambdaTransformer,+ FirehoseRedshiftDelivery redshiftDelivery) {+ this(storageFactory, s3Service, kinesisService, regionResolver, clock, config, parquetConverter,+ lambdaTransformer, redshiftDelivery, null);+ }++ public FirehoseService(StorageFactory storageFactory, S3Service s3Service, KinesisService kinesisService,+ RegionResolver regionResolver, Clock clock, EmulatorConfig config,+ FirehoseParquetConverter parquetConverter,+ FirehoseLambdaTransformer lambdaTransformer,+ FirehoseRedshiftDelivery redshiftDelivery,+ Executor redshiftDeliveryExecutor) {+ this.redshiftDelivery = redshiftDelivery;+ this.redshiftDeliveryExecutor = redshiftDeliveryExecutor != null+ ? redshiftDeliveryExecutor+ : Executors.newSingleThreadExecutor(r -> {+ Thread t = new Thread(r, "firehose-redshift-delivery");+ t.setDaemon(true);+ return t;+ });this.streamStore = storageFactory.create("firehose", "streams.json",new TypeReference<Map<String, DeliveryStreamDescription>>() {});this.sourceIteratorStore = storageFactory.create("firehose", "source-iterators.json",@@ -235,10 +270,29 @@ public class FirehoseService implements ResourceProvider {// persist them to disk via storageFactory.flushAll(). Each flush also commits the// source checkpoint it just made durable, so a graceful shutdown neither loses// records nor re-delivers them; storageFactory.flushAll() then writes that- // checkpoint out with everything else.+ // checkpoint out with everything else. Redshift COPY tasks are allowed a brief+ // window to complete before the delivery executor is forced down.void onPreShutdown(@Observes ShutdownDelayInitiatedEvent ignored) {flushExecutor.shutdownNow();buffers.keySet().forEach(this::flushByKey);+ if (redshiftDeliveryExecutor instanceof ExecutorService es) {+ es.shutdown();+ try {+ if (!es.awaitTermination(5, TimeUnit.SECONDS)) {+ dropAndLogQueuedTasks(es);+ }+ } catch (InterruptedException e) {+ dropAndLogQueuedTasks(es);+ Thread.currentThread().interrupt();+ }+ }+ }++ private static void dropAndLogQueuedTasks(ExecutorService es) {+ List<Runnable> dropped = es.shutdownNow();+ if (!dropped.isEmpty()) {+ LOG.warnv("Dropping {0} queued Redshift COPY task(s) on Firehose shutdown", dropped.size());+ }}/**@@ -1070,11 +1124,16 @@ public class FirehoseService implements ResourceProvider {}}- byte[] body = compression.compress(payload.toByteArray());+ byte[] rawPayload = payload.toByteArray();+ byte[] body = compression.compress(rawPayload);s3Service.putObject(bucket, key, body, "application/octet-stream", Map.of(),new PutObjectOptions().withContentEncoding(compression.contentEncoding()));LOG.infov("Flushed {0} records from stream {1} to s3://{2}/{3} ({4})",records.size(), streamName, bucket, key, compression.wireValue());+ if (isRedshiftStream(stream)) {+ deliverRedshiftBatch(streamName, stream, bucket, key, rawPayload, checkpoint);+ return;+ }// Only now: the records these iterators were read past are durable.commitSourceIterators(streamName, checkpoint);} catch (Exception e) {@@ -1084,6 +1143,79 @@ public class FirehoseService implements ResourceProvider {}}+ /**+ * Everything after a Redshift batch is staged in S3. The batch is durable in S3, so no failure here may hand its+ * records back to the buffer: that would stage and load them a second time. Backups and source checkpoints are+ * completed first, then COPY is submitted to run on its own thread so that a slow or hanging query never holds+ * the buffer flusher thread or PutRecord.+ */+ private void deliverRedshiftBatch(String streamName, DeliveryStreamDescription stream,+ String bucket, String key, byte[] rawPayload,+ Map<String, String> checkpoint) {+ RedshiftDestination redshift = stream.getDestinations().get(0).getRedshiftDestinationDescription();+ try {+ writeRedshiftBackup(stream, redshift, rawPayload);+ } catch (IOException | RuntimeException e) {+ LOG.errorv(e, "Backup write for stream {0} failed; the batch is staged in S3 and loads normally but "+ + "has no backup copy: {1}", streamName, describe(e));+ }+ try {+ commitSourceIterators(streamName, checkpoint);+ } catch (RuntimeException e) {+ LOG.errorv(e, "Could not persist the source checkpoint of stream {0} after its batch was staged; the "+ + "source records may be read again: {1}", streamName, describe(e));+ }+ if (redshiftDelivery == null) {+ LOG.warnv("Stream {0} has a Redshift destination but Redshift delivery is unavailable; "+ + "the batch stays in s3://{1}/{2}", streamName, bucket, key);+ return;+ }+ try {+ redshiftDeliveryExecutor.execute(() -> {+ RequestScopes.runAs(stream.getAccountId(), () -> {+ try {+ String region = AwsArnUtils.regionOrDefault(stream.getDeliveryStreamARN(),+ regionResolver.getDefaultRegion());+ redshiftDelivery.copy(stream.getAccountId(), region, redshift, bucket, key);+ } catch (Exception e) {+ LOG.errorv("Redshift delivery of s3://{0}/{1} into table {2} failed for stream {3}; "+ + "the batch stays in s3://{0}/{1}: {4}", bucket, key,+ redshift.getCopyCommand().getDataTableName(), streamName, describe(e));+ }+ });+ });+ } catch (RejectedExecutionException e) {+ LOG.errorv("Redshift delivery executor rejected task for s3://{0}/{1} into table {2} on stream {3}; "+ + "the batch stays in s3://{0}/{1}: {4}", bucket, key,+ redshift.getCopyCommand().getDataTableName(), streamName, describe(e));+ }+ }++ private static String describe(Exception e) {+ return e.getMessage() != null ? e.getMessage() : e.toString();+ }++ /** Writes the raw batch to the backup location under that location's own prefix, compression and bucket. */+ private void writeRedshiftBackup(DeliveryStreamDescription stream, RedshiftDestination redshift,+ byte[] rawPayload) throws IOException {+ S3Destination backup = redshift.getS3Backup();+ if (!"Enabled".equals(redshift.getS3BackupMode()) || backup == null || backup.bucketName() == null) {+ return;+ }+ FirehoseCompression compression = FirehoseCompression.forDelivery(backup.getCompressionFormat());+ String backupKey = S3ObjectKeyResolver.resolveKey(backup, stream.getDeliveryStreamName(),+ stream.getVersionId(), clock.instant(), compression);+ ensureBucket(backup.bucketName(), stream);+ s3Service.putObject(backup.bucketName(), backupKey, compression.compress(rawPayload),+ "application/octet-stream", Map.of(),+ new PutObjectOptions().withContentEncoding(compression.contentEncoding()));+ }++ private static boolean isRedshiftStream(DeliveryStreamDescription stream) {+ return stream.getDestinations() != null && !stream.getDestinations().isEmpty()+ && stream.getDestinations().get(0).getRedshiftDestinationDescription() != null;+ }+private void restoreFailedFlush(String streamName, List<byte[]> toFlush, Instant since,Map<String, String> checkpoint) {List<byte[]> buffer = buffers.get(streamName);diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySql.java b/src/main/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySql.javaindex 94ba37864e..bf881dc259 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySql.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySql.java@@ -1,5 +1,7 @@package io.github.hectorvent.floci.services.firehose;+import java.util.ArrayList;+import java.util.List;import java.util.regex.Pattern;/**@@ -10,15 +12,52 @@ import java.util.regex.Pattern;* outside a quoted literal, which is what keeps the text from carrying a second statement. The separator scan follows PostgreSQL's own* quoting rules rather than a simplified one, because the statement is later run in simple query mode, where* PostgreSQL executes every {@code ;}-separated statement it receives.+ *+ * <p>{@link #build} adds no {@code IAM_ROLE} or credentials. AWS hands the cluster temporary credentials of the+ * stream's role; the wire proxy reads a staged object straight from Floci's S3 when the statement names no role,+ * whereas an {@code IAM_ROLE} would have to exist in IAM and be associated with the cluster.*/final class RedshiftCopySql {private static final String NAME = "(?:[A-Za-z_][A-Za-z0-9_$]*|\"[^\"]+\")";private static final Pattern QUALIFIED_NAME = Pattern.compile(NAME + "(?:\\." + NAME + ")?");private static final Pattern SIMPLE_NAME = Pattern.compile(NAME);+ private static final Pattern BUCKET = Pattern.compile("[^/'\"\\s]+");+ private static final Pattern OBJECT_KEY = Pattern.compile("[^'\"\\s]*");private RedshiftCopySql() {}+ static String build(String table, String columns, String bucket, String key, String copyOptions) {+ if (!isValidTableName(table)) {+ throw new IllegalArgumentException("DataTableName is not a table identifier: " + table);+ }+ if (!isValidColumnList(columns)) {+ throw new IllegalArgumentException("DataTableColumns is not a list of column identifiers: " + columns);+ }+ if (!BUCKET.matcher(bucket).matches() || !OBJECT_KEY.matcher(key).matches()) {+ throw new IllegalArgumentException("The staged object s3://" + bucket + "/" + key+ + " has a name the COPY emulation cannot read: keys must not contain whitespace or quotes.");+ }+ if (copyOptions != null && hasStatementSeparator(copyOptions)) {+ throw new IllegalArgumentException("CopyOptions must not contain a statement separator.");+ }+ StringBuilder sql = new StringBuilder("COPY ").append(table);+ if (columns != null && !columns.isBlank()) {+ List<String> trimmed = new ArrayList<>();+ for (String column : columns.split(",")) {+ trimmed.add(column.trim());+ }+ sql.append(" (").append(String.join(", ", trimmed)).append(')');+ }+ sql.append(" FROM 's3://").append(bucket).append('/').append(key).append('\'');+ // The closing terminator is dropped: the statement is sent on its own, and a second one cannot follow it.+ String options = copyOptions == null ? "" : withoutTrailingTerminator(copyOptions).trim();+ if (!options.isEmpty()) {+ sql.append(' ').append(options);+ }+ return sql.toString();+ }+static boolean isValidTableName(String table) {return table != null && QUALIFIED_NAME.matcher(table).matches();}diff --git a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.javaindex 7057497e69..ff12e09cca 100644--- a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java+++ b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.DatabaseSecretCredentials;import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.common.SqlParameterParser.ParsedSql;import io.github.hectorvent.floci.services.rds.container.RdsBackendGate;@@ -196,7 +197,7 @@ public class RdsDataService implements Resettable {requiredText(request, "secretArn");RdsDataResourceResolver.DatabaseTarget target =resourceResolver.resolve(resourceArn, region);- Credentials credentials = credentials(request, target, region);+ DatabaseSecretCredentials credentials = credentials(request, target, region);String database = databaseName(request, target);// The transaction's connection keeps the cluster awake until it commits, rolls back or expires.@@ -302,7 +303,7 @@ public class RdsDataService implements Resettable {RdsDataResourceResolver.DatabaseTarget target =resourceResolver.resolve(resourceArn, region);- Credentials credentials = credentials(request, target, region);+ DatabaseSecretCredentials credentials = credentials(request, target, region);String database = databaseName(request, target);try (RdsBackendGate.Lease _ = enterBackend(target);Connection connection = connectionFactory.open(target, credentials.username(), credentials.password(), database)) {@@ -619,7 +620,7 @@ public class RdsDataService implements Resettable {};}- private Credentials credentials(JsonNode request, RdsDataResourceResolver.DatabaseTarget target, String region) {+ private DatabaseSecretCredentials credentials(JsonNode request, RdsDataResourceResolver.DatabaseTarget target, String region) {String secretArn = textOrNull(request, "secretArn");if (secretArn != null && !secretArn.isBlank()) {SecretVersion secret;@@ -628,7 +629,7 @@ public class RdsDataService implements Resettable {} catch (AwsException e) {throw new AwsException("SecretsErrorException", e.getMessage(), 400);}- Credentials fromSecret = parseSecretCredentials(secret.getSecretString());+ DatabaseSecretCredentials fromSecret = parseSecretCredentials(secret.getSecretString());if (fromSecret != null) {return fromSecret;}@@ -636,29 +637,23 @@ public class RdsDataService implements Resettable {"The secret must contain username and password fields.", 400);}String username = target.username() != null && !target.username().isBlank() ? target.username() : "root";- return new Credentials(username, target.password());+ return new DatabaseSecretCredentials(username, target.password());}- private Credentials parseSecretCredentials(String secretString) {+ private DatabaseSecretCredentials parseSecretCredentials(String secretString) {if (secretString == null || secretString.isBlank()) {return null;}+ JsonNode secret;try {- JsonNode secret = objectMapper.readTree(secretString);- String username = textOrNull(secret, "username");- if (username == null) {- username = textOrNull(secret, "user");- }- String password = textOrNull(secret, "password");- if (username != null && password != null) {- return new Credentials(username, password);- }+ secret = objectMapper.readTree(secretString);} catch (Exception e) {throw new AwsException("InvalidSecretException","The secret must contain valid JSON credentials.", 400);}- throw new AwsException("InvalidSecretException",- "The secret must contain username and password fields.", 400);+ return DatabaseSecretCredentials.parse(secret).orElseThrow(() ->+ new AwsException("InvalidSecretException",+ "The secret must contain username and password fields.", 400));}private String databaseName(JsonNode request, RdsDataResourceResolver.DatabaseTarget target) {@@ -892,9 +887,6 @@ public class RdsDataService implements Resettable {}}- private record Credentials(String username, String password) {- }-@FunctionalInterfaceprivate interface ConnectionWork {ObjectNode execute(Connection connection, DatabaseEngine engine) throws SQLException;diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.javaindex 33e1cffccd..7cc0bbe8e2 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.DatabaseSecretCredentials;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.services.redshift.RedshiftCredentialBroker;import io.github.hectorvent.floci.services.redshift.RedshiftService;@@ -79,12 +80,12 @@ class RedshiftDataResourceResolver {return new DatabaseTarget(workgroup.arn(), workgroup.host(), workgroup.port(), database,workgroup.masterUsername(), workgroup.masterPassword(), spectrum);}- Credentials creds = secretCredentials(request.get("SecretArn").asText(), region);+ DatabaseSecretCredentials creds = secretCredentials(request.get("SecretArn").asText(), region);return new DatabaseTarget(workgroup.arn(), workgroup.host(), workgroup.port(), database,creds.username(), creds.password(), spectrum);}- private Credentials secretCredentials(String secretArn, String region) {+ private DatabaseSecretCredentials secretCredentials(String secretArn, String region) {validateSecretRegion(secretArn, region);SecretVersion secret;try {@@ -92,11 +93,8 @@ class RedshiftDataResourceResolver {} catch (AwsException e) {throw validation("SecretArn does not resolve to a local secret: " + secretArn);}- Credentials creds = parseCredentials(secret.getSecretString());- if (creds == null) {- throw validation("Secret " + secretArn + " does not contain username and password fields.");- }- return creds;+ return DatabaseSecretCredentials.parse(objectMapper, secret.getSecretString())+ .orElseThrow(() -> validation("Secret " + secretArn + " does not contain username and password fields."));}private static void validateSecretRegion(String secretArn, String region) {@@ -115,7 +113,7 @@ class RedshiftDataResourceResolver {validateSecretRegion(secretArn, region);String clusterId = requiredText(request, "ClusterIdentifier");Cluster cluster = cluster(clusterId);- Credentials creds = secretCredentials(secretArn, region);+ DatabaseSecretCredentials creds = secretCredentials(secretArn, region);return target(clusterArn(clusterId, region), cluster, database, creds.username(), creds.password());}@@ -158,26 +156,6 @@ class RedshiftDataResourceResolver {return new DatabaseTarget(arn, host, port, database, user, password, spectrum);}- private Credentials parseCredentials(String secretString) {- if (secretString == null || secretString.isBlank()) {- return null;- }- try {- JsonNode node = objectMapper.readTree(secretString);- String username = text(node, "username");- if (username == null) {- username = text(node, "user");- }- String password = text(node, "password");- if (username != null && password != null) {- return new Credentials(username, password);- }- } catch (Exception e) {- LOG.debugv("Could not parse Redshift Data API secret: {0}", e.getMessage());- }- return null;- }-private String clusterArn(String clusterId, String region) {String r = (region == null || region.isBlank()) ? regionResolver.getDefaultRegion() : region;return AwsArnUtils.Arn.of("redshift", r, regionResolver.getAccountId(), "cluster:" + clusterId).toString();@@ -212,6 +190,4 @@ class RedshiftDataResourceResolver {}}- private record Credentials(String username, String password) {- }}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentialsTest.java b/src/test/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentialsTest.javanew file mode 100644index 0000000000..28a66a7e18--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentialsTest.java@@ -0,0 +1,78 @@+package io.github.hectorvent.floci.core.common;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import org.junit.jupiter.api.Test;++import java.util.Optional;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++class DatabaseSecretCredentialsTest {++ private final ObjectMapper mapper = new ObjectMapper();++ @Test+ void parsesUsernameAndPassword() {+ Optional<DatabaseSecretCredentials> creds = DatabaseSecretCredentials.parse(+ "{\"username\":\"admin\",\"password\":\"secret123\"}");++ assertTrue(creds.isPresent());+ assertEquals("admin", creds.get().username());+ assertEquals("secret123", creds.get().password());+ }++ @Test+ void fallsBackToUserFieldWhenUsernameIsAbsent() {+ Optional<DatabaseSecretCredentials> creds = DatabaseSecretCredentials.parse(+ "{\"user\":\"postgres\",\"password\":\"secret456\"}");++ assertTrue(creds.isPresent());+ assertEquals("postgres", creds.get().username());+ assertEquals("secret456", creds.get().password());+ }++ @Test+ void parsesWithCustomMapper() {+ Optional<DatabaseSecretCredentials> creds = DatabaseSecretCredentials.parse(+ mapper, "{\"username\":\"admin\",\"password\":\"secret\"}");++ assertTrue(creds.isPresent());+ assertEquals("admin", creds.get().username());+ assertEquals("secret", creds.get().password());+ }++ @Test+ void returnsEmptyForNullOrBlank() {+ assertTrue(DatabaseSecretCredentials.parse((String) null).isEmpty());+ assertTrue(DatabaseSecretCredentials.parse((JsonNode) null).isEmpty());+ assertTrue(DatabaseSecretCredentials.parse("").isEmpty());+ assertTrue(DatabaseSecretCredentials.parse(" ").isEmpty());+ }++ @Test+ void returnsEmptyForMalformedJson() {+ assertTrue(DatabaseSecretCredentials.parse("not-a-json").isEmpty());+ }++ @Test+ void returnsEmptyWhenMissingRequiredFields() {+ assertTrue(DatabaseSecretCredentials.parse("{\"username\":\"admin\"}").isEmpty());+ assertTrue(DatabaseSecretCredentials.parse("{\"password\":\"secret\"}").isEmpty());+ assertTrue(DatabaseSecretCredentials.parse("{\"other\":\"value\"}").isEmpty());+ }++ @Test+ void returnsEmptyForNonObjectNode() throws Exception {+ assertTrue(DatabaseSecretCredentials.parse(mapper.readTree("[\"item\"]")).isEmpty());+ }++ @Test+ void constructorRejectsNull() {+ assertThrows(NullPointerException.class, () -> new DatabaseSecretCredentials(null, "pass"));+ assertThrows(NullPointerException.class, () -> new DatabaseSecretCredentials("user", null));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryIntegrationTest.javanew file mode 100644index 0000000000..771b210ada--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryIntegrationTest.java@@ -0,0 +1,117 @@+package io.github.hectorvent.floci.services.firehose;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.services.firehose.model.Record;+import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;+import io.github.hectorvent.floci.services.redshift.RedshiftConnections;+import io.github.hectorvent.floci.services.redshift.RedshiftService;+import io.github.hectorvent.floci.services.redshift.model.Cluster;+import io.github.hectorvent.floci.testing.DockerAvailability;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.awaitility.Awaitility;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.Assumptions;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.nio.charset.StandardCharsets;+import java.sql.Connection;+import java.sql.ResultSet;+import java.sql.SQLException;+import java.sql.Statement;+import java.time.Duration;+import java.util.List;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * End to end: a record put into a Firehose stream with a Redshift destination is staged in S3 and+ * loaded into the cluster table through COPY. Needs a Docker daemon, like every Redshift+ * integration test.+ */+@QuarkusTest+class FirehoseRedshiftDeliveryIntegrationTest {++ @Inject+ FirehoseService firehoseService;++ @Inject+ RedshiftService redshiftService;++ @Inject+ ObjectMapper mapper;++ private String clusterId;++ @BeforeAll+ static void requireDocker() throws Exception {+ Assumptions.assumeTrue(DockerAvailability.isAvailable(),+ "Docker daemon must be available for Firehose Redshift delivery integration tests");+ }++ @AfterEach+ void cleanUp() {+ if (clusterId != null) {+ redshiftService.deleteCluster(clusterId);+ }+ }++ private RedshiftDestination destination(String clusterName, String table, String stageBucket) throws Exception {+ return mapper.readValue("""+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",+ "ClusterJDBCURL": "jdbc:redshift://%s.abc123.us-east-1.redshift.amazonaws.com:5439/dev",+ "Username": "admin", "Password": "Secret123",+ "CopyCommand": {"DataTableName": "%s", "CopyOptions": "json 'auto'"},+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose",+ "BucketARN": "arn:aws:s3:::%s"}}+ """.formatted(clusterName, table, stageBucket), RedshiftDestination.class);+ }++ @Test+ void putRecordIsLoadedIntoTheRedshiftTable() throws Exception {+ clusterId = "fh-rs-e2e";+ Cluster cluster = redshiftService.createCluster(clusterId, "dc2.large", "admin", "Secret123");+ try (Connection connection = RedshiftConnections.waitForConnection(cluster); Statement statement = connection.createStatement()) {+ statement.execute("CREATE TABLE fh_events (id int, name text)");+ }+ firehoseService.createDeliveryStream("us-east-1", "fh-to-redshift", null,+ destination(clusterId, "fh_events", "fh-rs-stage"), List.of(), null, null);+ firehoseService.putRecord("fh-to-redshift",+ new Record("{\"id\":1,\"name\":\"alice\"}".getBytes(StandardCharsets.UTF_8)));++ firehoseService.flush("fh-to-redshift");++ Awaitility.await()+ .atMost(Duration.ofSeconds(10))+ .pollInterval(Duration.ofMillis(200))+ .untilAsserted(() -> {+ try (Connection connection = RedshiftConnections.waitForConnection(cluster);+ Statement statement = connection.createStatement();+ ResultSet rows = statement.executeQuery("SELECT id, name FROM fh_events")) {+ assertTrue(rows.next(), "expected the delivered row in fh_events");+ assertEquals(1, rows.getInt("id"));+ assertEquals("alice", rows.getString("name"));+ assertFalse(rows.next());+ }+ });+ }++ @Test+ void missingTableKeepsTheStagedObjectAndTheRecord() throws Exception {+ clusterId = "fh-rs-no-table";+ redshiftService.createCluster(clusterId, "dc2.large", "admin", "Secret123");+ firehoseService.createDeliveryStream("us-east-1", "fh-no-table", null,+ destination(clusterId, "does_not_exist", "fh-rs-no-table-stage"), List.of(), null, null);+ firehoseService.putRecord("fh-no-table",+ new Record("{\"id\":1,\"name\":\"alice\"}".getBytes(StandardCharsets.UTF_8)));++ firehoseService.flush("fh-no-table");++ String listing = given().when().get("/fh-rs-no-table-stage").then().statusCode(200).extract().asString();+ assertTrue(listing.contains("<Key>"), "expected the staged object, got: " + listing);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryTest.javanew file mode 100644index 0000000000..a23e438304--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryTest.java@@ -0,0 +1,170 @@+package io.github.hectorvent.floci.services.firehose;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;+import io.github.hectorvent.floci.services.redshift.RedshiftService;+import io.github.hectorvent.floci.services.redshift.model.Cluster;+import io.github.hectorvent.floci.services.redshift.model.Endpoint;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;+import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.sql.Connection;+import java.sql.SQLException;+import java.sql.Statement;+import java.util.ArrayList;+import java.util.List;+import java.util.Properties;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class FirehoseRedshiftDeliveryTest {++ private final ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();+ private RedshiftService redshift;+ private SecretsManagerService secrets;+ private final List<String[]> executed = new ArrayList<>();+ private FirehoseRedshiftDelivery delivery;++ @BeforeEach+ void setUp() {+ redshift = mock(RedshiftService.class);+ secrets = mock(SecretsManagerService.class);+ Cluster cluster = mock(Cluster.class);+ when(cluster.getEndpoint()).thenReturn(new Endpoint("host.docker.internal", 41234));+ when(cluster.getMasterUsername()).thenReturn("admin");+ when(redshift.describeClustersForAccount("000000000000", "c1")).thenReturn(List.of(cluster));+ delivery = new FirehoseRedshiftDelivery(redshift, secrets, mapper,+ (url, user, password, sql) -> executed.add(new String[] {url, user, password, sql}));+ }++ private RedshiftDestination destination() throws Exception {+ return mapper.readValue("""+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",+ "Username": "admin", "Password": "Secret123",+ "CopyCommand": {"DataTableName": "events", "CopyOptions": "json 'auto'"}}+ """, RedshiftDestination.class);+ }++ @Test+ void runsCopyWithoutAnIamRoleThroughTheProxyPortInSimpleMode() throws Exception {+ delivery.copy("000000000000", "us-east-1", destination(), "stage", "k1");++ assertEquals(1, executed.size());+ assertEquals("jdbc:postgresql://127.0.0.1:41234/dev?preferQueryMode=simple", executed.get(0)[0]);+ assertEquals("admin", executed.get(0)[1]);+ assertEquals("Secret123", executed.get(0)[2]);+ assertEquals("COPY events FROM 's3://stage/k1' json 'auto'", executed.get(0)[3]);+ }++ @Test+ void secretOverridesInlineCredentialsWhenEnabled() throws Exception {+ RedshiftDestination dest = destination();+ RedshiftDestination.SecretsManagerConfiguration cfg = new RedshiftDestination.SecretsManagerConfiguration();+ cfg.setEnabled(true);+ cfg.setSecretArn("arn:aws:secretsmanager:us-east-1:000000000000:secret:rs-creds-AbCdEf");+ dest.setSecretsManagerConfiguration(cfg);+ SecretVersion version = mock(SecretVersion.class);+ when(version.getSecretString()).thenReturn("{\"username\":\"fromsecret\",\"password\":\"pw-secret\"}");+ when(secrets.getSecretValue(eq(cfg.getSecretArn()), any(), any(), eq("us-east-1"))).thenReturn(version);++ delivery.copy("000000000000", "us-east-1", dest, "stage", "k1");++ assertEquals("fromsecret", executed.get(0)[1]);+ assertEquals("pw-secret", executed.get(0)[2]);+ }++ @Test+ void disabledSecretsManagerKeepsTheInlineCredentials() throws Exception {+ RedshiftDestination dest = destination();+ RedshiftDestination.SecretsManagerConfiguration cfg = new RedshiftDestination.SecretsManagerConfiguration();+ cfg.setEnabled(false);+ dest.setSecretsManagerConfiguration(cfg);++ delivery.copy("000000000000", "us-east-1", dest, "stage", "k1");++ assertEquals("Secret123", executed.get(0)[2]);+ }++ @Test+ void unknownClusterSurfacesAsAwsException() {+ when(redshift.describeClustersForAccount("000000000000", "c1"))+ .thenThrow(new AwsException("ClusterNotFound", "Cluster c1 not found", 404));++ assertThrows(AwsException.class,+ () -> delivery.copy("000000000000", "us-east-1", destination(), "stage", "k"));+ }++ @Test+ void jdbcFailureIsWrappedWithTableClusterAndSqlState() {+ FirehoseRedshiftDelivery failing = new FirehoseRedshiftDelivery(redshift, secrets, mapper,+ (url, user, password, sql) -> {+ throw new SQLException("relation \"events\" does not exist", "42P01");+ });++ IllegalStateException e = assertThrows(IllegalStateException.class,+ () -> failing.copy("000000000000", "us-east-1", destination(), "stage", "k"));++ assertTrue(e.getMessage().contains("events"), e.getMessage());+ assertTrue(e.getMessage().contains("c1"), e.getMessage());+ assertTrue(e.getMessage().contains("42P01"), e.getMessage());+ }++ @Test+ void jdbcConnectionsCarryConnectAndSocketTimeouts() {+ Properties properties = FirehoseRedshiftDelivery.connectionProperties("admin", "Secret123");++ assertEquals("admin", properties.getProperty("user"));+ assertEquals("Secret123", properties.getProperty("password"));+ assertTrue(Integer.parseInt(properties.getProperty("connectTimeout")) > 0);+ assertTrue(Integer.parseInt(properties.getProperty("socketTimeout")) > 0);+ }++ @Test+ void closeFailureAfterACommittedCopyIsNotAFailedDelivery() throws Exception {+ Connection connection = mock(Connection.class);+ Statement statement = mock(Statement.class);+ when(connection.createStatement()).thenReturn(statement);+ doThrow(new SQLException("connection reset")).when(connection).close();++ FirehoseRedshiftDelivery.executeAndClose(connection, "COPY t FROM 's3://b/k'");++ verify(statement).execute("COPY t FROM 's3://b/k'");+ }++ @Test+ void failedCopyIsReportedAndTheConnectionStillClosed() throws Exception {+ Connection connection = mock(Connection.class);+ Statement statement = mock(Statement.class);+ when(connection.createStatement()).thenReturn(statement);+ when(statement.execute(any())).thenThrow(new SQLException("relation does not exist", "42P01"));++ SQLException e = assertThrows(SQLException.class,+ () -> FirehoseRedshiftDelivery.executeAndClose(connection, "COPY t FROM 's3://b/k'"));++ assertEquals("42P01", e.getSQLState());+ verify(connection).close();+ }++ @Test+ void destinationWithoutAnyPasswordFailsWithAClearMessage() throws Exception {+ RedshiftDestination dest = destination();+ dest.setPassword(null);++ IllegalStateException e = assertThrows(IllegalStateException.class,+ () -> delivery.copy("000000000000", "us-east-1", dest, "stage", "k"));++ assertTrue(e.getMessage().contains("Password"), e.getMessage());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDestinationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDestinationIntegrationTest.javaindex 31f732df47..4cdf14d8f1 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDestinationIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDestinationIntegrationTest.java@@ -233,7 +233,7 @@ class FirehoseRedshiftDestinationIntegrationTest {}@Test- void flushStagesTheBatchInTheIntermediateBucket() {+ void flushWithAMissingClusterKeepsTheStagedObject() {post("CreateDeliveryStream", createBody("rs-flush", "rs-flush-stage")).statusCode(200);String data = Base64.getEncoder().encodeToString("{\"id\":1}".getBytes(StandardCharsets.UTF_8));post("PutRecord", "{\"DeliveryStreamName\": \"rs-flush\", \"Record\": {\"Data\": \"" + data + "\"}}")diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.javanew file mode 100644index 0000000000..b21dd10bf6--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java@@ -0,0 +1,279 @@+package io.github.hectorvent.floci.services.firehose;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.firehose.model.Record;+import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;+import io.github.hectorvent.floci.services.kinesis.KinesisService;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;+import org.mockito.InOrder;+import org.mockito.Mockito;++import java.nio.charset.StandardCharsets;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.concurrent.Executor;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.RejectedExecutionException;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyLong;+import static org.mockito.ArgumentMatchers.anyMap;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class FirehoseRedshiftServiceTest {++ private static final String STREAM = "rs-delivery";+ private static final String BACKUP_ENABLED = ", \"S3BackupMode\": \"Enabled\", \"S3BackupConfiguration\": "+ + "{\"RoleARN\": \"arn:aws:iam::000000000000:role/firehose\", \"BucketARN\": \"arn:aws:s3:::bk\"}";++ private final ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();+ private FirehoseService service;+ private S3Service s3Service;+ private FirehoseRedshiftDelivery delivery;+ private MutableClock clock;+ private StorageFactory storageFactory;+ private RegionResolver regionResolver;+ private EmulatorConfig config;++ @BeforeEach+ void setUp() {+ storageFactory = Mockito.mock(StorageFactory.class);+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();+ when(storageFactory.create(anyString(), anyString(), any()))+ .thenAnswer(invocation -> backends.computeIfAbsent(+ invocation.getArgument(0) + "/" + invocation.getArgument(1),+ k -> AccountAwareStorageBackend.inMemory("000000000000")));+ s3Service = Mockito.mock(S3Service.class);+ delivery = Mockito.mock(FirehoseRedshiftDelivery.class);+ clock = new MutableClock();++ EmulatorConfig.FirehoseServiceConfig firehoseCfg = mock(EmulatorConfig.FirehoseServiceConfig.class);+ when(firehoseCfg.enabled()).thenReturn(true);+ when(firehoseCfg.tickIntervalSeconds()).thenReturn(10L);+ when(firehoseCfg.flushRecordCount()).thenReturn(0);+ EmulatorConfig.ServicesConfig servicesCfg = mock(EmulatorConfig.ServicesConfig.class);+ when(servicesCfg.firehose()).thenReturn(firehoseCfg);+ config = mock(EmulatorConfig.class);+ when(config.services()).thenReturn(servicesCfg);+ regionResolver = new RegionResolver("us-east-1", "000000000000");++ service = new FirehoseService(storageFactory, s3Service, new KinesisService(storageFactory, regionResolver),+ regionResolver, clock, config, mock(FirehoseParquetConverter.class),+ mock(FirehoseLambdaTransformer.class), delivery, Runnable::run);+ }++ private void createStream(String extraMembers) throws Exception {+ RedshiftDestination destination = mapper.readValue("""+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",+ "Username": "admin", "Password": "Secret123",+ "CopyCommand": {"DataTableName": "events"},+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}+ %s}+ """.formatted(extraMembers), RedshiftDestination.class);+ service.createDeliveryStream("us-east-1", STREAM, null, destination, List.of(), null, null);+ service.putRecord(STREAM, new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));+ }++ private void copyFails() {+ doThrow(new IllegalStateException("relation does not exist"))+ .when(delivery).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ private void verifyStagedTimes(int times) {+ verify(s3Service, times(times)).putObject(eq("stage"), anyString(), any(byte[].class), anyString(),+ anyMap(), any());+ }++ @Test+ void successfulCopyConsumesTheBatchOnce() throws Exception {+ createStream("");++ service.flush(STREAM);+ service.flush(STREAM);++ verify(delivery, times(1)).copy(eq("000000000000"), eq("us-east-1"), any(), eq("stage"), anyString());+ verifyStagedTimes(1);+ }++ @Test+ void failedCopyIsLoggedAndNotRestagedOrRebuffered() throws Exception {+ createStream("");+ copyFails();++ service.flush(STREAM);+ service.flush(STREAM);++ verify(delivery, times(1)).copy(anyString(), anyString(), any(), eq("stage"), anyString());+ verifyStagedTimes(1);+ }++ @Test+ void deliveryUnavailableLeavesStagedObjectInS3() throws Exception {+ FirehoseService serviceWithoutDelivery = new FirehoseService(storageFactory, s3Service,+ new KinesisService(storageFactory, regionResolver), regionResolver, clock, config,+ mock(FirehoseParquetConverter.class), mock(FirehoseLambdaTransformer.class), null, Runnable::run);+ RedshiftDestination destination = mapper.readValue("""+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",+ "Username": "admin", "Password": "Secret123",+ "CopyCommand": {"DataTableName": "events"},+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}}+ """, RedshiftDestination.class);+ serviceWithoutDelivery.createDeliveryStream("us-east-1", "fh-no-delivery", null, destination, List.of(), null, null);+ serviceWithoutDelivery.putRecord("fh-no-delivery", new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));++ serviceWithoutDelivery.flush("fh-no-delivery");++ verifyStagedTimes(1);+ verify(delivery, never()).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void backupEnabledWritesTheBatchToTheBackupBucket() throws Exception {+ createStream(BACKUP_ENABLED);++ service.flush(STREAM);++ verifyStagedTimes(1);+ verify(s3Service, times(1)).putObject(eq("bk"), anyString(), any(byte[].class), anyString(), anyMap(), any());+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), eq("stage"), anyString());+ }++ @Test+ void backupObjectUsesTheBackupConfigurationPrefix() throws Exception {+ createStream(", \"S3BackupMode\": \"Enabled\", \"S3BackupConfiguration\": "+ + "{\"RoleARN\": \"arn:aws:iam::000000000000:role/firehose\", \"BucketARN\": \"arn:aws:s3:::bk\","+ + " \"Prefix\": \"backup/\"}");++ service.flush(STREAM);++ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);+ verify(s3Service).putObject(eq("bk"), keys.capture(), any(byte[].class), anyString(), anyMap(), any());+ assertTrue(keys.getValue().startsWith("backup/"), keys.getValue());+ }++ @Test+ void aFailingBackupWriteStillLoadsTheStagedBatchAndDoesNotRestageIt() throws Exception {+ createStream(BACKUP_ENABLED);+ doThrow(new IllegalStateException("backup bucket unavailable")).when(s3Service)+ .putObject(eq("bk"), anyString(), any(byte[].class), anyString(), anyMap(), any());++ service.flush(STREAM);+ service.flush(STREAM);++ verifyStagedTimes(1);+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void aFailingPrimaryWriteWritesNoBackupSoNothingIsDelivered() throws Exception {+ createStream(BACKUP_ENABLED);+ doThrow(new IllegalStateException("s3 unavailable")).when(s3Service)+ .putObject(eq("stage"), anyString(), any(byte[].class), anyString(), anyMap(), any());++ service.flush(STREAM);++ verify(s3Service, never()).putObject(eq("bk"), anyString(), any(byte[].class), anyString(), anyMap(), any());+ verify(delivery, never()).copy(anyString(), anyString(), any(), anyString(), anyString());+ }++ @Test+ void rejectedExecutionDoesNotRestoreRecordsToBuffer() throws Exception {+ Executor rejectingExecutor = r -> {+ throw new RejectedExecutionException("executor shut down");+ };+ FirehoseService rejectingService = new FirehoseService(storageFactory, s3Service,+ new KinesisService(storageFactory, regionResolver), regionResolver, clock, config,+ mock(FirehoseParquetConverter.class), mock(FirehoseLambdaTransformer.class), delivery, rejectingExecutor);++ RedshiftDestination destination = mapper.readValue("""+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",+ "Username": "admin", "Password": "Secret123",+ "CopyCommand": {"DataTableName": "events"},+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}}+ """, RedshiftDestination.class);+ rejectingService.createDeliveryStream("us-east-1", "fh-rejected", null, destination, List.of(), null, null);+ rejectingService.putRecord("fh-rejected", new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));++ rejectingService.flush("fh-rejected");+ rejectingService.flush("fh-rejected");++ verify(s3Service, times(1)).putObject(eq("stage"), anyString(), any(byte[].class), anyString(),+ anyMap(), any());+ }++ @Test+ void onPreShutdownFlushesBuffersAndRunsCopyBeforeShuttingDown() throws Exception {+ ExecutorService executor = Executors.newSingleThreadExecutor();+ FirehoseService shutdownService = new FirehoseService(storageFactory, s3Service,+ new KinesisService(storageFactory, regionResolver), regionResolver, clock, config,+ mock(FirehoseParquetConverter.class), mock(FirehoseLambdaTransformer.class), delivery, executor);++ RedshiftDestination destination = mapper.readValue("""+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",+ "Username": "admin", "Password": "Secret123",+ "CopyCommand": {"DataTableName": "events"},+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}}+ """, RedshiftDestination.class);+ shutdownService.createDeliveryStream("us-east-1", "fh-shutdown", null, destination, List.of(), null, null);+ shutdownService.putRecord("fh-shutdown", new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));++ shutdownService.onPreShutdown(null);++ verify(s3Service, times(1)).putObject(eq("stage"), anyString(), any(byte[].class), anyString(),+ anyMap(), any());+ verify(delivery, times(1)).copy(eq("000000000000"), eq("us-east-1"), any(), eq("stage"), anyString());+ assertTrue(executor.isTerminated());+ }++ @Test+ void onPreShutdownDropsQueuedTasksWhenTerminationTimesOut() throws Exception {+ ExecutorService mockExecutor = mock(ExecutorService.class);+ FirehoseService shutdownService = new FirehoseService(storageFactory, s3Service,+ new KinesisService(storageFactory, regionResolver), regionResolver, clock, config,+ mock(FirehoseParquetConverter.class), mock(FirehoseLambdaTransformer.class), delivery, mockExecutor);++ RedshiftDestination destination = mapper.readValue("""+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",+ "Username": "admin", "Password": "Secret123",+ "CopyCommand": {"DataTableName": "events"},+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}}+ """, RedshiftDestination.class);+ shutdownService.createDeliveryStream("us-east-1", "fh-timeout", null, destination, List.of(), null, null);+ shutdownService.putRecord("fh-timeout", new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));++ when(mockExecutor.awaitTermination(anyLong(), any())).thenReturn(false);+ when(mockExecutor.shutdownNow()).thenReturn(List.of(() -> {}));++ shutdownService.onPreShutdown(null);++ InOrder inOrder = Mockito.inOrder(mockExecutor);+ inOrder.verify(mockExecutor).execute(any());+ inOrder.verify(mockExecutor).shutdown();+ inOrder.verify(mockExecutor).awaitTermination(5, TimeUnit.SECONDS);+ inOrder.verify(mockExecutor).shutdownNow();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySqlTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySqlTest.javaindex 8f0a7e960c..9ff89dea34 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySqlTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySqlTest.java@@ -4,6 +4,7 @@ import org.junit.jupiter.api.Test;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;class RedshiftCopySqlTest {@@ -118,6 +119,50 @@ class RedshiftCopySqlTest {assertTrue(RedshiftCopySql.hasStatementSeparator("NULL AS $x"));}+ @Test+ void buildsCopyWithOptionsAndNoIamRole() {+ String sql = RedshiftCopySql.build("events", null, "stage", "2026/10/19/f.gz", "json 'auto' gzip");++ assertEquals("COPY events FROM 's3://stage/2026/10/19/f.gz' json 'auto' gzip", sql);+ }++ @Test+ void buildDropsTheClosingTerminatorOfTheApiReferenceExample() {+ String sql = RedshiftCopySql.build("t", null, "b", "k", "delimiter '\\t' lzop;");++ assertEquals("COPY t FROM 's3://b/k' delimiter '\\t' lzop", sql);+ }++ @Test+ void buildKeepsATerminatorInsideAQuotedValue() {+ String sql = RedshiftCopySql.build("t", null, "b", "k", "DELIMITER ';' CSV");++ assertEquals("COPY t FROM 's3://b/k' DELIMITER ';' CSV", sql);+ }++ @Test+ void tableAndColumnsAreKeptVerbatimSoRedshiftCaseFoldingApplies() {+ String sql = RedshiftCopySql.build("Public.MyEvents", "Id, \"Full Name\"", "stage", "k", null);++ assertEquals("COPY Public.MyEvents (Id, \"Full Name\") FROM 's3://stage/k'", sql);+ }++ @Test+ void buildRejectsNamesAndOptionsThatCouldCarryAnotherStatement() {+ assertThrows(IllegalArgumentException.class, () -> RedshiftCopySql.build("t; DROP TABLE x", null, "b", "k", null));+ assertThrows(IllegalArgumentException.class,+ () -> RedshiftCopySql.build("t", "id, name); DROP TABLE t; --", "b", "k", null));+ assertThrows(IllegalArgumentException.class, () -> RedshiftCopySql.build("t", null, "b", "k", "gzip; DROP TABLE t"));+ }++ @Test+ void objectKeyThatTheProxyCouldNotParseIsRejected() {+ for (String key : new String[] {"my prefix/f.gz", "it's/f.gz", "say\"/f.gz"}) {+ assertThrows(IllegalArgumentException.class,+ () -> RedshiftCopySql.build("t", null, "stage", key, null), key);+ }+ }+@Testvoid dollarQuotingAndCommentsAreRefused() {assertTrue(RedshiftCopySql.hasStatementSeparator("NULL AS $$'$$ gzip"));diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftConnections.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftConnections.javanew file mode 100644index 0000000000..55c5eade00--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftConnections.java@@ -0,0 +1,45 @@+package io.github.hectorvent.floci.services.redshift;++import io.github.hectorvent.floci.services.redshift.model.Cluster;+import org.awaitility.Awaitility;+import org.awaitility.core.ConditionTimeoutException;++import java.sql.Connection;+import java.sql.DriverManager;+import java.sql.SQLException;+import java.time.Duration;+import java.util.Objects;++public final class RedshiftConnections {++ private RedshiftConnections() {}++ public static String jdbcUrl(Cluster cluster) {+ return "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort() + "/dev";+ }++ public static Connection waitForConnection(Cluster cluster) throws SQLException {+ return waitForConnection(cluster, "admin", "Secret123");+ }++ public static Connection waitForConnection(Cluster cluster, String username, String password) throws SQLException {+ return waitForConnection(jdbcUrl(cluster), username, password);+ }++ public static Connection waitForConnection(String host, int port, String username, String password) throws SQLException {+ return waitForConnection("jdbc:postgresql://" + host + ":" + port + "/dev", username, password);+ }++ public static Connection waitForConnection(String jdbcUrl, String username, String password) throws SQLException {+ try {+ return Awaitility.await()+ .atMost(Duration.ofSeconds(30))+ .pollDelay(Duration.ZERO)+ .pollInterval(Duration.ofMillis(500))+ .ignoreExceptions()+ .until(() -> DriverManager.getConnection(jdbcUrl, username, password), Objects::nonNull);+ } catch (ConditionTimeoutException e) {+ return DriverManager.getConnection(jdbcUrl, username, password);+ }+ }+}