Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(firehose): deliver staged Redshift batches with single-attempt COPY (#6006)

Vũ Thế Huyfd37e109f8c8
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseRedshiftTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseRedshiftTest.java
new file mode 100644
index 0000000000..be519f5b29
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseRedshiftTest.java
@@ -0,0 +1,183 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.core.SdkBytes;
+import software.amazon.awssdk.services.firehose.FirehoseClient;
+import software.amazon.awssdk.services.firehose.model.BufferingHints;
+import software.amazon.awssdk.services.firehose.model.CopyCommand;
+import software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamRequest;
+import software.amazon.awssdk.services.firehose.model.DeleteDeliveryStreamRequest;
+import software.amazon.awssdk.services.firehose.model.DescribeDeliveryStreamRequest;
+import software.amazon.awssdk.services.firehose.model.DestinationDescription;
+import software.amazon.awssdk.services.firehose.model.PutRecordRequest;
+import software.amazon.awssdk.services.firehose.model.Record;
+import software.amazon.awssdk.services.firehose.model.RedshiftDestinationConfiguration;
+import software.amazon.awssdk.services.firehose.model.ResourceNotFoundException;
+import software.amazon.awssdk.services.firehose.model.S3DestinationConfiguration;
+import software.amazon.awssdk.services.redshift.RedshiftClient;
+import software.amazon.awssdk.services.redshift.model.ClusterNotFoundException;
+import software.amazon.awssdk.services.redshift.model.CreateClusterRequest;
+import software.amazon.awssdk.services.redshift.model.DeleteClusterRequest;
+import software.amazon.awssdk.services.redshiftdata.RedshiftDataClient;
+import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementRequest;
+import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementResponse;
+import software.amazon.awssdk.services.redshiftdata.model.ExecuteStatementRequest;
+import software.amazon.awssdk.services.redshiftdata.model.GetStatementResultRequest;
+import software.amazon.awssdk.services.redshiftdata.model.StatusString;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@DisplayName("Firehose Redshift destination")
+class FirehoseRedshiftTest {
+
+ private static final String STREAM = "sdk-rs-shape-" + UUID.randomUUID().toString().substring(0, 8);
+ private static final String JDBC_URL = "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev";
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/firehose";
+ private static FirehoseClient firehose;
+
+ @BeforeAll
+ static void setUp() {
+ firehose = TestFixtures.firehoseClient();
+ }
+
+ @AfterAll
+ static void tearDown() {
+ if (firehose != null) {
+ try {
+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder().deliveryStreamName(STREAM).build());
+ } catch (ResourceNotFoundException ignored) {
+ // The stream does not exist when a test failed before creating it.
+ }
+ firehose.close();
+ }
+ }
+
+ @Test
+ @DisplayName("CreateDeliveryStream stores the Redshift destination and DescribeDeliveryStream returns it")
+ void createAndDescribe() {
+ firehose.createDeliveryStream(CreateDeliveryStreamRequest.builder()
+ .deliveryStreamName(STREAM)
+ .redshiftDestinationConfiguration(RedshiftDestinationConfiguration.builder()
+ .roleARN(ROLE_ARN)
+ .clusterJDBCURL(JDBC_URL)
+ .username("admin")
+ .password("Secret123")
+ .copyCommand(CopyCommand.builder().dataTableName("events").copyOptions("json 'auto'").build())
+ .s3Configuration(S3DestinationConfiguration.builder()
+ .roleARN(ROLE_ARN)
+ .bucketARN("arn:aws:s3:::sdk-rs-stage")
+ .build())
+ .build())
+ .build());
+
+ DestinationDescription destination = firehose.describeDeliveryStream(
+ DescribeDeliveryStreamRequest.builder().deliveryStreamName(STREAM).build())
+ .deliveryStreamDescription().destinations().get(0);
+
+ assertThat(destination.redshiftDestinationDescription().clusterJDBCURL()).isEqualTo(JDBC_URL);
+ assertThat(destination.redshiftDestinationDescription().username()).isEqualTo("admin");
+ assertThat(destination.redshiftDestinationDescription().copyCommand().dataTableName()).isEqualTo("events");
+ assertThat(destination.redshiftDestinationDescription().s3DestinationDescription().bucketARN())
+ .isEqualTo("arn:aws:s3:::sdk-rs-stage");
+ }
+
+ @Test
+ @DisplayName("PutRecord is loaded into the Redshift table through COPY")
+ void deliversIntoTheRedshiftTable() throws Exception {
+ String clusterId = TestFixtures.uniqueName("fh-rs-cluster");
+ String stream = TestFixtures.uniqueName("fh-rs-delivery");
+ String table = "fh_events";
+ RedshiftClient redshift = TestFixtures.redshiftClient();
+ RedshiftDataClient data = TestFixtures.redshiftDataClient();
+ try {
+ redshift.createCluster(CreateClusterRequest.builder()
+ .clusterIdentifier(clusterId)
+ .nodeType("dc2.large")
+ .masterUsername("admin")
+ .masterUserPassword("password123")
+ .build());
+ awaitFinished(data, execute(data, clusterId, "CREATE TABLE " + table + " (id int, name varchar(20))"));
+
+ firehose.createDeliveryStream(CreateDeliveryStreamRequest.builder()
+ .deliveryStreamName(stream)
+ .redshiftDestinationConfiguration(RedshiftDestinationConfiguration.builder()
+ .roleARN(ROLE_ARN)
+ .clusterJDBCURL("jdbc:redshift://" + clusterId
+ + ".abc123.us-east-1.redshift.amazonaws.com:5439/dev")
+ .username("admin")
+ .password("password123")
+ .copyCommand(CopyCommand.builder()
+ .dataTableName(table).copyOptions("json 'auto'").build())
+ .s3Configuration(S3DestinationConfiguration.builder()
+ .roleARN(ROLE_ARN)
+ .bucketARN("arn:aws:s3:::" + stream + "-stage")
+ .bufferingHints(BufferingHints.builder()
+ .intervalInSeconds(60).sizeInMBs(1).build())
+ .build())
+ .build())
+ .build());
+ firehose.putRecord(PutRecordRequest.builder()
+ .deliveryStreamName(stream)
+ .record(Record.builder().data(SdkBytes.fromString("{\"id\":1,\"name\":\"alice\"}",
+ StandardCharsets.UTF_8)).build())
+ .build());
+
+ Instant deadline = Instant.now().plus(Duration.ofSeconds(150));
+ long rows = 0;
+ while (rows == 0 && Instant.now().isBefore(deadline)) {
+ String id = awaitFinished(data, execute(data, clusterId, "SELECT count(*) FROM " + table));
+ rows = data.getStatementResult(GetStatementResultRequest.builder().id(id).build())
+ .records().get(0).get(0).longValue();
+ if (rows == 0) {
+ Thread.sleep(5000);
+ }
+ }
+ assertThat(rows).isEqualTo(1L);
+ } finally {
+ try {
+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder().deliveryStreamName(stream).build());
+ } catch (ResourceNotFoundException ignored) {
+ // The stream does not exist when creation failed.
+ }
+ try {
+ redshift.deleteCluster(DeleteClusterRequest.builder().clusterIdentifier(clusterId).build());
+ } catch (ClusterNotFoundException ignored) {
+ // The cluster does not exist when creation failed.
+ }
+ data.close();
+ redshift.close();
+ }
+ }
+
+ private static String execute(RedshiftDataClient data, String clusterId, String sql) {
+ return data.executeStatement(ExecuteStatementRequest.builder()
+ .clusterIdentifier(clusterId)
+ .dbUser("admin")
+ .database("dev")
+ .sql(sql)
+ .build()).id();
+ }
+
+ private static String awaitFinished(RedshiftDataClient data, String id) throws InterruptedException {
+ Instant deadline = Instant.now().plus(Duration.ofSeconds(30));
+ while (Instant.now().isBefore(deadline)) {
+ DescribeStatementResponse describe = data.describeStatement(DescribeStatementRequest.builder().id(id).build());
+ if (describe.status() == StatusString.FINISHED) {
+ return id;
+ }
+ if (describe.status() == StatusString.FAILED || describe.status() == StatusString.ABORTED) {
+ throw new IllegalStateException("Statement " + id + " ended " + describe.status() + ": " + describe.error());
+ }
+ Thread.sleep(250);
+ }
+ throw new IllegalStateException("Statement " + id + " did not finish in time");
+ }
+}
diff --git a/docs/services/firehose.md b/docs/services/firehose.md
index ca8a01a8fb..0bf758ef3d 100644
--- a/docs/services/firehose.md
+++ b/docs/services/firehose.md
@@ -142,17 +142,52 @@ quoted value a `;`, a `$` (dollar quoting is never needed) and the comment marke
unterminated quote. One closing `;` at the very end is the exception (the API reference's own example,
`delimiter '\t' lzop;`, is accepted). Create and update reject anything else.
-This release accepts and stores the destination and stages each batch in the intermediate S3 location, like an
-S3 destination. It does not run the `COPY` into the cluster yet, and `S3BackupMode` has no effect on what is written.
+### Delivery
+
+A flush writes the batch to the intermediate S3 location, then runs
+`COPY <DataTableName> [(<DataTableColumns>)] FROM 's3://<bucket>/<key>' <CopyOptions>` against the cluster. A single
+closing `;` in `CopyOptions` is dropped from the statement. The `COPY` names no `IAM_ROLE`: AWS gives the cluster
+temporary credentials of the stream's `RoleARN`, and Floci instead lets the proxy read the staged object from its own
+S3, so `RoleARN` is validated and stored but not used for the load (an `IAM_ROLE` would have to exist in IAM and be
+associated with the cluster). The `COPY` goes through the cluster's wire proxy, so it behaves like a `COPY` sent by any
+other client (see [Redshift COPY from S3](redshift.md#copy-from-s3)). Credentials come from `Username` and `Password`,
+or from the secret in `SecretsManagerConfiguration` when `Enabled` is true (the secret JSON needs `username` and
+`password`). An empty `DataTableColumns` omits the column list, so the proxy discovers the table columns.
+
+`DataTableName` and `DataTableColumns` are passed to the `COPY` as written, so Redshift's identifier rules apply
+(an unquoted name is lowercased, a quoted one is kept).
+
+A batch is staged in S3 once, and the `COPY` runs on its own thread so that a slow or hanging query does not hold the
+buffer flusher thread or `PutRecord`. If its `COPY` fails (missing table or cluster, bad data), the failure is logged
+and the staged object stays in S3; retrying failed loads will follow in a separate change. With `S3BackupMode` set to
+`Enabled`, which needs an `S3BackupConfiguration`, each batch is also written once to that bucket under its own prefix
+and compression. The backup is written after the batch is staged, and a failed backup write is logged without holding
+the load back or re-staging the batch, so that batch then has no backup copy.
+
+A JDBC connection that stops answering is dropped after 150 seconds.
+
+Compression of the intermediate objects is not added to the `COPY` for you: as on AWS, put `GZIP` in `CopyOptions`
+when the intermediate `CompressionFormat` is `GZIP`.
+
+Floci does not write the `redshift_failed` manifest objects that AWS creates after a failed `COPY`.
### Known deviations from AWS
- `ClusterJDBCURL` must match the AWS pattern, so it needs a host such as
`mycluster.abc123.us-east-1.redshift.amazonaws.com`. Floci treats the URL as a logical address: the first
host label is the cluster identifier, and the host suffix and port are ignored. The database name is ignored
- too, because the emulated database is always `dev`. The cluster is not looked up when the stream is created, so a stream
- can be created before its cluster.
+ too, because the emulated database is always `dev`. The cluster is looked up at delivery time, so a stream can
+ be created before its cluster.
- `ProcessingConfiguration` on a Redshift destination is accepted and returned but not executed.
+- A `ClusterJDBCURL` with a `redshift-serverless` host is accepted, but delivery looks the name up as a provisioned
+ cluster, so a serverless workgroup is not a delivery target yet.
+- The `COPY` carries no credentials, so the proxy reads the staged object as an anonymous client. With S3
+ authorization enforced and a private intermediate bucket the load fails and the staged object stays in S3; use a
+ bucket the anonymous principal can read, or leave enforcement off.
+- `CopyOptions` are validated for shape only. The `COPY` emulation does not implement every Redshift option
+ (`LZOP`, `BZIP2`, `ZSTD`, `PARQUET` and others): such a statement is forwarded to PostgreSQL and fails, and the staged
+ object stays in S3; retrying failed loads will follow in a separate change. Firehose never writes an `LZOP` object,
+ so the API reference's `lzop` example would fail on AWS too.
- The intermediate `Prefix` must not contain whitespace or quotes, because the `COPY` emulation cannot read such
object keys. `DataFormatConversionConfiguration` is ignored on the Redshift `S3Configuration`, which AWS does not
define.
diff --git a/docs/services/redshift.md b/docs/services/redshift.md
index c1b2020a99..61c0e84c3c 100644
--- a/docs/services/redshift.md
+++ b/docs/services/redshift.md
@@ -442,6 +442,9 @@ order) through its own S3 service and streams the rows into the backing PostgreS
bind parameters. Zero-parameter JDBC `PreparedStatement` calls therefore work with pgjdbc's
default extended mode. Statements containing bind parameters are forwarded unchanged.
+Firehose delivery streams with a Redshift destination use this same path: Firehose stages each batch in S3 and
+runs the `COPY` over a connection to the cluster proxy. See [Firehose](firehose.md#redshift-destination).
+
### Limitations
- DDL rewriting works in both Simple Query (`'Q'`) and Extended Query (`Parse`) flows. COPY and
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentials.java b/src/main/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentials.java
new file mode 100644
index 0000000000..3d75d30c8c
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentials.java
@@ -0,0 +1,70 @@
+package io.github.hectorvent.floci.core.common;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.jboss.logging.Logger;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Username and password credentials stored in an AWS Secrets Manager secret for a database (RDS,
+ * Redshift, Redshift Data API, or Firehose Redshift destination).
+ */
+public record DatabaseSecretCredentials(String username, String password) {
+
+ private static final Logger LOG = Logger.getLogger(DatabaseSecretCredentials.class);
+ private static final ObjectMapper DEFAULT_MAPPER = new ObjectMapper();
+
+ public DatabaseSecretCredentials {
+ Objects.requireNonNull(username, "username must not be null");
+ Objects.requireNonNull(password, "password must not be null");
+ }
+
+ /**
+ * Parses credentials from a Secrets Manager secret string JSON payload using the default mapper.
+ */
+ public static Optional<DatabaseSecretCredentials> parse(String secretString) {
+ return parse(DEFAULT_MAPPER, secretString);
+ }
+
+ /**
+ * Parses credentials from a Secrets Manager secret string using the given ObjectMapper.
+ */
+ public static Optional<DatabaseSecretCredentials> parse(ObjectMapper mapper, String secretString) {
+ if (secretString == null || secretString.isBlank() || mapper == null) {
+ return Optional.empty();
+ }
+ try {
+ return parse(mapper.readTree(secretString));
+ } catch (Exception ignored) {
+ // An unparseable secret string is treated as missing credentials; callers handle Optional.empty()
+ LOG.debugv("Failed to parse database credentials from secret string: {0}", ignored.getMessage());
+ return Optional.empty();
+ }
+ }
+
+ /**
+ * Parses credentials from a Secrets Manager secret JSON node.
+ * Looks for non-null "username" (or fallback "user") and "password" text fields.
+ */
+ public static Optional<DatabaseSecretCredentials> parse(JsonNode node) {
+ if (node == null || !node.isObject()) {
+ return Optional.empty();
+ }
+ String username = textOrNull(node, "username");
+ if (username == null) {
+ username = textOrNull(node, "user");
+ }
+ String password = textOrNull(node, "password");
+ if (username != null && password != null) {
+ return Optional.of(new DatabaseSecretCredentials(username, password));
+ }
+ return Optional.empty();
+ }
+
+ private static String textOrNull(JsonNode node, String field) {
+ JsonNode child = node.get(field);
+ return (child != null && !child.isNull()) ? child.asText() : null;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDelivery.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDelivery.java
new file mode 100644
index 0000000000..9fd7f06621
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDelivery.java
@@ -0,0 +1,150 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.DatabaseSecretCredentials;
+import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;
+import io.github.hectorvent.floci.services.redshift.RedshiftService;
+import io.github.hectorvent.floci.services.redshift.model.Cluster;
+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
+import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+
+/**
+ * Loads a delivered S3 object into a Floci Redshift cluster. The {@code COPY} runs over a JDBC
+ * connection to the cluster's wire proxy in simple query mode, so the proxy's S3 COPY emulation
+ * does the work exactly as it does for any other client.
+ */
+@ApplicationScoped
+public class FirehoseRedshiftDelivery {
+
+ private static final Logger LOG = Logger.getLogger(FirehoseRedshiftDelivery.class);
+ private static final int CONNECT_TIMEOUT_SECONDS = 10;
+ private static final int SOCKET_TIMEOUT_SECONDS = 150;
+
+ /** Seam for tests; production opens a real JDBC connection. */
+ interface SqlRunner {
+ void execute(String jdbcUrl, String user, String password, String sql) throws SQLException;
+ }
+
+ private final RedshiftService redshiftService;
+ private final SecretsManagerService secretsManagerService;
+ private final ObjectMapper mapper;
+ private final SqlRunner runner;
+
+ @Inject
+ public FirehoseRedshiftDelivery(RedshiftService redshiftService, SecretsManagerService secretsManagerService,
+ ObjectMapper mapper) {
+ this(redshiftService, secretsManagerService, mapper, FirehoseRedshiftDelivery::runJdbc);
+ }
+
+ FirehoseRedshiftDelivery(RedshiftService redshiftService, SecretsManagerService secretsManagerService,
+ ObjectMapper mapper, SqlRunner runner) {
+ this.redshiftService = redshiftService;
+ this.secretsManagerService = secretsManagerService;
+ this.mapper = mapper;
+ this.runner = runner;
+ }
+
+ /**
+ * Runs the destination's COPY for the object at {@code s3://bucket/key}.
+ *
+ * Throws {@code AwsException} when the cluster or the secret cannot be resolved.
+ *
+ * @throws IllegalStateException when the statement itself fails; the message carries the table and SQL state
+ */
+ public void copy(String accountId, String region, RedshiftDestination destination, String bucket, String key) {
+ RedshiftJdbcUrl target = RedshiftJdbcUrl.parse(destination.getClusterJdbcUrl());
+ Cluster cluster = redshiftService.describeClustersForAccount(accountId, target.clusterIdentifier()).get(0);
+ String[] credentials = credentials(destination, cluster, region);
+ RedshiftDestination.CopyCommand command = destination.getCopyCommand();
+ String sql = RedshiftCopySql.build(command.getDataTableName(), command.getDataTableColumns(), bucket, key,
+ command.getCopyOptions());
+ // The database name in the URL is ignored: the emulated container database is always "dev".
+ String jdbcUrl = "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort()
+ + "/dev?preferQueryMode=simple";
+ try {
+ runner.execute(jdbcUrl, credentials[0], credentials[1], sql);
+ } catch (SQLException e) {
+ throw new IllegalStateException("COPY into " + command.getDataTableName() + " on cluster "
+ + target.clusterIdentifier() + " failed (SQLSTATE " + e.getSQLState() + "): " + e.getMessage(), e);
+ }
+ LOG.debugv("Firehose copied s3://{0}/{1} into {2} on cluster {3}", bucket, key,
+ command.getDataTableName(), target.clusterIdentifier());
+ }
+
+ private String[] credentials(RedshiftDestination destination, Cluster cluster, String region) {
+ RedshiftDestination.SecretsManagerConfiguration secrets = destination.getSecretsManagerConfiguration();
+ if (secrets != null && Boolean.TRUE.equals(secrets.getEnabled())) {
+ SecretVersion secret = secretsManagerService.getSecretValue(secrets.getSecretArn(), null, null, region);
+ DatabaseSecretCredentials credentials = DatabaseSecretCredentials.parse(mapper, secret.getSecretString())
+ .orElseThrow(() -> new IllegalStateException("Secret " + secrets.getSecretArn()
+ + " does not contain username and password fields."));
+ return new String[] {credentials.username(), credentials.password()};
+ }
+ if (destination.getPassword() == null) {
+ throw new IllegalStateException("The Redshift destination for cluster " + cluster.getClusterIdentifier()
+ + " has no Password and no enabled SecretsManagerConfiguration, so Firehose cannot sign in.");
+ }
+ String user = destination.getUsername() != null ? destination.getUsername() : cluster.getMasterUsername();
+ return new String[] {user, destination.getPassword()};
+ }
+
+ /**
+ * Timeouts matter here: a connection that never answers would stall the Redshift delivery thread
+ * and delay emulator shutdown.
+ */
+ static Properties connectionProperties(String user, String password) {
+ Properties properties = new Properties();
+ properties.setProperty("user", user);
+ properties.setProperty("password", password);
+ properties.setProperty("connectTimeout", String.valueOf(CONNECT_TIMEOUT_SECONDS));
+ properties.setProperty("socketTimeout", String.valueOf(SOCKET_TIMEOUT_SECONDS));
+ return properties;
+ }
+
+ private static void runJdbc(String jdbcUrl, String user, String password, String sql) throws SQLException {
+ Connection connection = DriverManager.getConnection(jdbcUrl, connectionProperties(user, password));
+ executeAndClose(connection, sql);
+ }
+
+ /**
+ * Runs the statement and closes the connection. Once the statement has returned the load is committed
+ * (autocommit), so a failure while closing must not surface as a failed delivery.
+ */
+ static void executeAndClose(Connection connection, String sql) throws SQLException {
+ try {
+ // Closing the connection closes its statement, so the statement is not closed on its own: a failure there
+ // would be reported as a failed COPY too.
+ Statement statement = connection.createStatement();
+ statement.execute(sql);
+ } catch (SQLException e) {
+ closeAfterFailure(connection, e);
+ throw e;
+ }
+ closeAfterSuccess(connection);
+ }
+
+ private static void closeAfterFailure(Connection connection, SQLException failure) {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ failure.addSuppressed(e);
+ }
+ }
+
+ private static void closeAfterSuccess(Connection connection) {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ LOG.warnv(e, "Closing the Redshift connection after a committed COPY failed: {0}", e.getMessage());
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 6959acdf75..9f85cb7e70 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -30,16 +30,21 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.time.Clock;
import java.time.Instant;
-import java.util.*;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -112,6 +117,8 @@ public class FirehoseService implements ResourceProvider {
private final Map<String, Map<String, String>> pendingSourceIterators = new ConcurrentHashMap<>();
private final FirehoseParquetConverter parquetConverter;
private final FirehoseLambdaTransformer lambdaTransformer;
+ private final FirehoseRedshiftDelivery redshiftDelivery;
+ private final Executor redshiftDeliveryExecutor;
// Serialises the read, version check, merge and write of a Redshift UpdateDestination per stream.
private final Map<String, Object> redshiftUpdateLocks = new ConcurrentHashMap<>();
private final Clock clock;
@@ -196,11 +203,39 @@ public class FirehoseService implements ResourceProvider {
sourceIteratorStore.deleteForAccount(accountFromScopedKey(key), logicalKeyFromScopedKey(key));
}
- @Inject
+ /** Kept so callers that never deliver to Redshift need no delivery collaborator; null means unavailable. */
public FirehoseService(StorageFactory storageFactory, S3Service s3Service, KinesisService kinesisService,
RegionResolver regionResolver, Clock clock, EmulatorConfig config,
FirehoseParquetConverter parquetConverter,
FirehoseLambdaTransformer lambdaTransformer) {
+ this(storageFactory, s3Service, kinesisService, regionResolver, clock, config, parquetConverter,
+ lambdaTransformer, null);
+ }
+
+ @Inject
+ public FirehoseService(StorageFactory storageFactory, S3Service s3Service, KinesisService kinesisService,
+ RegionResolver regionResolver, Clock clock, EmulatorConfig config,
+ FirehoseParquetConverter parquetConverter,
+ FirehoseLambdaTransformer lambdaTransformer,
+ FirehoseRedshiftDelivery redshiftDelivery) {
+ this(storageFactory, s3Service, kinesisService, regionResolver, clock, config, parquetConverter,
+ lambdaTransformer, redshiftDelivery, null);
+ }
+
+ public FirehoseService(StorageFactory storageFactory, S3Service s3Service, KinesisService kinesisService,
+ RegionResolver regionResolver, Clock clock, EmulatorConfig config,
+ FirehoseParquetConverter parquetConverter,
+ FirehoseLambdaTransformer lambdaTransformer,
+ FirehoseRedshiftDelivery redshiftDelivery,
+ Executor redshiftDeliveryExecutor) {
+ this.redshiftDelivery = redshiftDelivery;
+ this.redshiftDeliveryExecutor = redshiftDeliveryExecutor != null
+ ? redshiftDeliveryExecutor
+ : Executors.newSingleThreadExecutor(r -> {
+ Thread t = new Thread(r, "firehose-redshift-delivery");
+ t.setDaemon(true);
+ return t;
+ });
this.streamStore = storageFactory.create("firehose", "streams.json",
new TypeReference<Map<String, DeliveryStreamDescription>>() {});
this.sourceIteratorStore = storageFactory.create("firehose", "source-iterators.json",
@@ -235,10 +270,29 @@ public class FirehoseService implements ResourceProvider {
// persist them to disk via storageFactory.flushAll(). Each flush also commits the
// source checkpoint it just made durable, so a graceful shutdown neither loses
// records nor re-delivers them; storageFactory.flushAll() then writes that
- // checkpoint out with everything else.
+ // checkpoint out with everything else. Redshift COPY tasks are allowed a brief
+ // window to complete before the delivery executor is forced down.
void onPreShutdown(@Observes ShutdownDelayInitiatedEvent ignored) {
flushExecutor.shutdownNow();
buffers.keySet().forEach(this::flushByKey);
+ if (redshiftDeliveryExecutor instanceof ExecutorService es) {
+ es.shutdown();
+ try {
+ if (!es.awaitTermination(5, TimeUnit.SECONDS)) {
+ dropAndLogQueuedTasks(es);
+ }
+ } catch (InterruptedException e) {
+ dropAndLogQueuedTasks(es);
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ private static void dropAndLogQueuedTasks(ExecutorService es) {
+ List<Runnable> dropped = es.shutdownNow();
+ if (!dropped.isEmpty()) {
+ LOG.warnv("Dropping {0} queued Redshift COPY task(s) on Firehose shutdown", dropped.size());
+ }
}
/**
@@ -1070,11 +1124,16 @@ public class FirehoseService implements ResourceProvider {
}
}
- byte[] body = compression.compress(payload.toByteArray());
+ byte[] rawPayload = payload.toByteArray();
+ byte[] body = compression.compress(rawPayload);
s3Service.putObject(bucket, key, body, "application/octet-stream", Map.of(),
new PutObjectOptions().withContentEncoding(compression.contentEncoding()));
LOG.infov("Flushed {0} records from stream {1} to s3://{2}/{3} ({4})",
records.size(), streamName, bucket, key, compression.wireValue());
+ if (isRedshiftStream(stream)) {
+ deliverRedshiftBatch(streamName, stream, bucket, key, rawPayload, checkpoint);
+ return;
+ }
// Only now: the records these iterators were read past are durable.
commitSourceIterators(streamName, checkpoint);
} catch (Exception e) {
@@ -1084,6 +1143,79 @@ public class FirehoseService implements ResourceProvider {
}
}
+ /**
+ * Everything after a Redshift batch is staged in S3. The batch is durable in S3, so no failure here may hand its
+ * records back to the buffer: that would stage and load them a second time. Backups and source checkpoints are
+ * completed first, then COPY is submitted to run on its own thread so that a slow or hanging query never holds
+ * the buffer flusher thread or PutRecord.
+ */
+ private void deliverRedshiftBatch(String streamName, DeliveryStreamDescription stream,
+ String bucket, String key, byte[] rawPayload,
+ Map<String, String> checkpoint) {
+ RedshiftDestination redshift = stream.getDestinations().get(0).getRedshiftDestinationDescription();
+ try {
+ writeRedshiftBackup(stream, redshift, rawPayload);
+ } catch (IOException | RuntimeException e) {
+ LOG.errorv(e, "Backup write for stream {0} failed; the batch is staged in S3 and loads normally but "
+ + "has no backup copy: {1}", streamName, describe(e));
+ }
+ try {
+ commitSourceIterators(streamName, checkpoint);
+ } catch (RuntimeException e) {
+ LOG.errorv(e, "Could not persist the source checkpoint of stream {0} after its batch was staged; the "
+ + "source records may be read again: {1}", streamName, describe(e));
+ }
+ if (redshiftDelivery == null) {
+ LOG.warnv("Stream {0} has a Redshift destination but Redshift delivery is unavailable; "
+ + "the batch stays in s3://{1}/{2}", streamName, bucket, key);
+ return;
+ }
+ try {
+ redshiftDeliveryExecutor.execute(() -> {
+ RequestScopes.runAs(stream.getAccountId(), () -> {
+ try {
+ String region = AwsArnUtils.regionOrDefault(stream.getDeliveryStreamARN(),
+ regionResolver.getDefaultRegion());
+ redshiftDelivery.copy(stream.getAccountId(), region, redshift, bucket, key);
+ } catch (Exception e) {
+ LOG.errorv("Redshift delivery of s3://{0}/{1} into table {2} failed for stream {3}; "
+ + "the batch stays in s3://{0}/{1}: {4}", bucket, key,
+ redshift.getCopyCommand().getDataTableName(), streamName, describe(e));
+ }
+ });
+ });
+ } catch (RejectedExecutionException e) {
+ LOG.errorv("Redshift delivery executor rejected task for s3://{0}/{1} into table {2} on stream {3}; "
+ + "the batch stays in s3://{0}/{1}: {4}", bucket, key,
+ redshift.getCopyCommand().getDataTableName(), streamName, describe(e));
+ }
+ }
+
+ private static String describe(Exception e) {
+ return e.getMessage() != null ? e.getMessage() : e.toString();
+ }
+
+ /** Writes the raw batch to the backup location under that location's own prefix, compression and bucket. */
+ private void writeRedshiftBackup(DeliveryStreamDescription stream, RedshiftDestination redshift,
+ byte[] rawPayload) throws IOException {
+ S3Destination backup = redshift.getS3Backup();
+ if (!"Enabled".equals(redshift.getS3BackupMode()) || backup == null || backup.bucketName() == null) {
+ return;
+ }
+ FirehoseCompression compression = FirehoseCompression.forDelivery(backup.getCompressionFormat());
+ String backupKey = S3ObjectKeyResolver.resolveKey(backup, stream.getDeliveryStreamName(),
+ stream.getVersionId(), clock.instant(), compression);
+ ensureBucket(backup.bucketName(), stream);
+ s3Service.putObject(backup.bucketName(), backupKey, compression.compress(rawPayload),
+ "application/octet-stream", Map.of(),
+ new PutObjectOptions().withContentEncoding(compression.contentEncoding()));
+ }
+
+ private static boolean isRedshiftStream(DeliveryStreamDescription stream) {
+ return stream.getDestinations() != null && !stream.getDestinations().isEmpty()
+ && stream.getDestinations().get(0).getRedshiftDestinationDescription() != null;
+ }
+
private void restoreFailedFlush(String streamName, List<byte[]> toFlush, Instant since,
Map<String, String> checkpoint) {
List<byte[]> buffer = buffers.get(streamName);
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySql.java b/src/main/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySql.java
index 94ba37864e..bf881dc259 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySql.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySql.java
@@ -1,5 +1,7 @@
package io.github.hectorvent.floci.services.firehose;
+import java.util.ArrayList;
+import java.util.List;
import java.util.regex.Pattern;
/**
@@ -10,15 +12,52 @@ import java.util.regex.Pattern;
* outside a quoted literal, which is what keeps the text from carrying a second statement. The separator scan follows PostgreSQL's own
* quoting rules rather than a simplified one, because the statement is later run in simple query mode, where
* PostgreSQL executes every {@code ;}-separated statement it receives.
+ *
+ * <p>{@link #build} adds no {@code IAM_ROLE} or credentials. AWS hands the cluster temporary credentials of the
+ * stream's role; the wire proxy reads a staged object straight from Floci's S3 when the statement names no role,
+ * whereas an {@code IAM_ROLE} would have to exist in IAM and be associated with the cluster.
*/
final class RedshiftCopySql {
private static final String NAME = "(?:[A-Za-z_][A-Za-z0-9_$]*|\"[^\"]+\")";
private static final Pattern QUALIFIED_NAME = Pattern.compile(NAME + "(?:\\." + NAME + ")?");
private static final Pattern SIMPLE_NAME = Pattern.compile(NAME);
+ private static final Pattern BUCKET = Pattern.compile("[^/'\"\\s]+");
+ private static final Pattern OBJECT_KEY = Pattern.compile("[^'\"\\s]*");
private RedshiftCopySql() {}
+ static String build(String table, String columns, String bucket, String key, String copyOptions) {
+ if (!isValidTableName(table)) {
+ throw new IllegalArgumentException("DataTableName is not a table identifier: " + table);
+ }
+ if (!isValidColumnList(columns)) {
+ throw new IllegalArgumentException("DataTableColumns is not a list of column identifiers: " + columns);
+ }
+ if (!BUCKET.matcher(bucket).matches() || !OBJECT_KEY.matcher(key).matches()) {
+ throw new IllegalArgumentException("The staged object s3://" + bucket + "/" + key
+ + " has a name the COPY emulation cannot read: keys must not contain whitespace or quotes.");
+ }
+ if (copyOptions != null && hasStatementSeparator(copyOptions)) {
+ throw new IllegalArgumentException("CopyOptions must not contain a statement separator.");
+ }
+ StringBuilder sql = new StringBuilder("COPY ").append(table);
+ if (columns != null && !columns.isBlank()) {
+ List<String> trimmed = new ArrayList<>();
+ for (String column : columns.split(",")) {
+ trimmed.add(column.trim());
+ }
+ sql.append(" (").append(String.join(", ", trimmed)).append(')');
+ }
+ sql.append(" FROM 's3://").append(bucket).append('/').append(key).append('\'');
+ // The closing terminator is dropped: the statement is sent on its own, and a second one cannot follow it.
+ String options = copyOptions == null ? "" : withoutTrailingTerminator(copyOptions).trim();
+ if (!options.isEmpty()) {
+ sql.append(' ').append(options);
+ }
+ return sql.toString();
+ }
+
static boolean isValidTableName(String table) {
return table != null && QUALIFIED_NAME.matcher(table).matches();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java
index 7057497e69..ff12e09cca 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataService.java
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.DatabaseSecretCredentials;
import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.common.SqlParameterParser.ParsedSql;
import io.github.hectorvent.floci.services.rds.container.RdsBackendGate;
@@ -196,7 +197,7 @@ public class RdsDataService implements Resettable {
requiredText(request, "secretArn");
RdsDataResourceResolver.DatabaseTarget target =
resourceResolver.resolve(resourceArn, region);
- Credentials credentials = credentials(request, target, region);
+ DatabaseSecretCredentials credentials = credentials(request, target, region);
String database = databaseName(request, target);
// The transaction's connection keeps the cluster awake until it commits, rolls back or expires.
@@ -302,7 +303,7 @@ public class RdsDataService implements Resettable {
RdsDataResourceResolver.DatabaseTarget target =
resourceResolver.resolve(resourceArn, region);
- Credentials credentials = credentials(request, target, region);
+ DatabaseSecretCredentials credentials = credentials(request, target, region);
String database = databaseName(request, target);
try (RdsBackendGate.Lease _ = enterBackend(target);
Connection connection = connectionFactory.open(target, credentials.username(), credentials.password(), database)) {
@@ -619,7 +620,7 @@ public class RdsDataService implements Resettable {
};
}
- private Credentials credentials(JsonNode request, RdsDataResourceResolver.DatabaseTarget target, String region) {
+ private DatabaseSecretCredentials credentials(JsonNode request, RdsDataResourceResolver.DatabaseTarget target, String region) {
String secretArn = textOrNull(request, "secretArn");
if (secretArn != null && !secretArn.isBlank()) {
SecretVersion secret;
@@ -628,7 +629,7 @@ public class RdsDataService implements Resettable {
} catch (AwsException e) {
throw new AwsException("SecretsErrorException", e.getMessage(), 400);
}
- Credentials fromSecret = parseSecretCredentials(secret.getSecretString());
+ DatabaseSecretCredentials fromSecret = parseSecretCredentials(secret.getSecretString());
if (fromSecret != null) {
return fromSecret;
}
@@ -636,29 +637,23 @@ public class RdsDataService implements Resettable {
"The secret must contain username and password fields.", 400);
}
String username = target.username() != null && !target.username().isBlank() ? target.username() : "root";
- return new Credentials(username, target.password());
+ return new DatabaseSecretCredentials(username, target.password());
}
- private Credentials parseSecretCredentials(String secretString) {
+ private DatabaseSecretCredentials parseSecretCredentials(String secretString) {
if (secretString == null || secretString.isBlank()) {
return null;
}
+ JsonNode secret;
try {
- JsonNode secret = objectMapper.readTree(secretString);
- String username = textOrNull(secret, "username");
- if (username == null) {
- username = textOrNull(secret, "user");
- }
- String password = textOrNull(secret, "password");
- if (username != null && password != null) {
- return new Credentials(username, password);
- }
+ secret = objectMapper.readTree(secretString);
} catch (Exception e) {
throw new AwsException("InvalidSecretException",
"The secret must contain valid JSON credentials.", 400);
}
- throw new AwsException("InvalidSecretException",
- "The secret must contain username and password fields.", 400);
+ return DatabaseSecretCredentials.parse(secret).orElseThrow(() ->
+ new AwsException("InvalidSecretException",
+ "The secret must contain username and password fields.", 400));
}
private String databaseName(JsonNode request, RdsDataResourceResolver.DatabaseTarget target) {
@@ -892,9 +887,6 @@ public class RdsDataService implements Resettable {
}
}
- private record Credentials(String username, String password) {
- }
-
@FunctionalInterface
private interface ConnectionWork {
ObjectNode execute(Connection connection, DatabaseEngine engine) throws SQLException;
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java
index 33e1cffccd..7cc0bbe8e2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.DatabaseSecretCredentials;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.services.redshift.RedshiftCredentialBroker;
import io.github.hectorvent.floci.services.redshift.RedshiftService;
@@ -79,12 +80,12 @@ class RedshiftDataResourceResolver {
return new DatabaseTarget(workgroup.arn(), workgroup.host(), workgroup.port(), database,
workgroup.masterUsername(), workgroup.masterPassword(), spectrum);
}
- Credentials creds = secretCredentials(request.get("SecretArn").asText(), region);
+ DatabaseSecretCredentials creds = secretCredentials(request.get("SecretArn").asText(), region);
return new DatabaseTarget(workgroup.arn(), workgroup.host(), workgroup.port(), database,
creds.username(), creds.password(), spectrum);
}
- private Credentials secretCredentials(String secretArn, String region) {
+ private DatabaseSecretCredentials secretCredentials(String secretArn, String region) {
validateSecretRegion(secretArn, region);
SecretVersion secret;
try {
@@ -92,11 +93,8 @@ class RedshiftDataResourceResolver {
} catch (AwsException e) {
throw validation("SecretArn does not resolve to a local secret: " + secretArn);
}
- Credentials creds = parseCredentials(secret.getSecretString());
- if (creds == null) {
- throw validation("Secret " + secretArn + " does not contain username and password fields.");
- }
- return creds;
+ return DatabaseSecretCredentials.parse(objectMapper, secret.getSecretString())
+ .orElseThrow(() -> validation("Secret " + secretArn + " does not contain username and password fields."));
}
private static void validateSecretRegion(String secretArn, String region) {
@@ -115,7 +113,7 @@ class RedshiftDataResourceResolver {
validateSecretRegion(secretArn, region);
String clusterId = requiredText(request, "ClusterIdentifier");
Cluster cluster = cluster(clusterId);
- Credentials creds = secretCredentials(secretArn, region);
+ DatabaseSecretCredentials creds = secretCredentials(secretArn, region);
return target(clusterArn(clusterId, region), cluster, database, creds.username(), creds.password());
}
@@ -158,26 +156,6 @@ class RedshiftDataResourceResolver {
return new DatabaseTarget(arn, host, port, database, user, password, spectrum);
}
- private Credentials parseCredentials(String secretString) {
- if (secretString == null || secretString.isBlank()) {
- return null;
- }
- try {
- JsonNode node = objectMapper.readTree(secretString);
- String username = text(node, "username");
- if (username == null) {
- username = text(node, "user");
- }
- String password = text(node, "password");
- if (username != null && password != null) {
- return new Credentials(username, password);
- }
- } catch (Exception e) {
- LOG.debugv("Could not parse Redshift Data API secret: {0}", e.getMessage());
- }
- return null;
- }
-
private String clusterArn(String clusterId, String region) {
String r = (region == null || region.isBlank()) ? regionResolver.getDefaultRegion() : region;
return AwsArnUtils.Arn.of("redshift", r, regionResolver.getAccountId(), "cluster:" + clusterId).toString();
@@ -212,6 +190,4 @@ class RedshiftDataResourceResolver {
}
}
- private record Credentials(String username, String password) {
- }
}
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentialsTest.java b/src/test/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentialsTest.java
new file mode 100644
index 0000000000..28a66a7e18
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/DatabaseSecretCredentialsTest.java
@@ -0,0 +1,78 @@
+package io.github.hectorvent.floci.core.common;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DatabaseSecretCredentialsTest {
+
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ @Test
+ void parsesUsernameAndPassword() {
+ Optional<DatabaseSecretCredentials> creds = DatabaseSecretCredentials.parse(
+ "{\"username\":\"admin\",\"password\":\"secret123\"}");
+
+ assertTrue(creds.isPresent());
+ assertEquals("admin", creds.get().username());
+ assertEquals("secret123", creds.get().password());
+ }
+
+ @Test
+ void fallsBackToUserFieldWhenUsernameIsAbsent() {
+ Optional<DatabaseSecretCredentials> creds = DatabaseSecretCredentials.parse(
+ "{\"user\":\"postgres\",\"password\":\"secret456\"}");
+
+ assertTrue(creds.isPresent());
+ assertEquals("postgres", creds.get().username());
+ assertEquals("secret456", creds.get().password());
+ }
+
+ @Test
+ void parsesWithCustomMapper() {
+ Optional<DatabaseSecretCredentials> creds = DatabaseSecretCredentials.parse(
+ mapper, "{\"username\":\"admin\",\"password\":\"secret\"}");
+
+ assertTrue(creds.isPresent());
+ assertEquals("admin", creds.get().username());
+ assertEquals("secret", creds.get().password());
+ }
+
+ @Test
+ void returnsEmptyForNullOrBlank() {
+ assertTrue(DatabaseSecretCredentials.parse((String) null).isEmpty());
+ assertTrue(DatabaseSecretCredentials.parse((JsonNode) null).isEmpty());
+ assertTrue(DatabaseSecretCredentials.parse("").isEmpty());
+ assertTrue(DatabaseSecretCredentials.parse(" ").isEmpty());
+ }
+
+ @Test
+ void returnsEmptyForMalformedJson() {
+ assertTrue(DatabaseSecretCredentials.parse("not-a-json").isEmpty());
+ }
+
+ @Test
+ void returnsEmptyWhenMissingRequiredFields() {
+ assertTrue(DatabaseSecretCredentials.parse("{\"username\":\"admin\"}").isEmpty());
+ assertTrue(DatabaseSecretCredentials.parse("{\"password\":\"secret\"}").isEmpty());
+ assertTrue(DatabaseSecretCredentials.parse("{\"other\":\"value\"}").isEmpty());
+ }
+
+ @Test
+ void returnsEmptyForNonObjectNode() throws Exception {
+ assertTrue(DatabaseSecretCredentials.parse(mapper.readTree("[\"item\"]")).isEmpty());
+ }
+
+ @Test
+ void constructorRejectsNull() {
+ assertThrows(NullPointerException.class, () -> new DatabaseSecretCredentials(null, "pass"));
+ assertThrows(NullPointerException.class, () -> new DatabaseSecretCredentials("user", null));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryIntegrationTest.java
new file mode 100644
index 0000000000..771b210ada
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryIntegrationTest.java
@@ -0,0 +1,117 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.services.firehose.model.Record;
+import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;
+import io.github.hectorvent.floci.services.redshift.RedshiftConnections;
+import io.github.hectorvent.floci.services.redshift.RedshiftService;
+import io.github.hectorvent.floci.services.redshift.model.Cluster;
+import io.github.hectorvent.floci.testing.DockerAvailability;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.awaitility.Awaitility;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.time.Duration;
+import java.util.List;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * End to end: a record put into a Firehose stream with a Redshift destination is staged in S3 and
+ * loaded into the cluster table through COPY. Needs a Docker daemon, like every Redshift
+ * integration test.
+ */
+@QuarkusTest
+class FirehoseRedshiftDeliveryIntegrationTest {
+
+ @Inject
+ FirehoseService firehoseService;
+
+ @Inject
+ RedshiftService redshiftService;
+
+ @Inject
+ ObjectMapper mapper;
+
+ private String clusterId;
+
+ @BeforeAll
+ static void requireDocker() throws Exception {
+ Assumptions.assumeTrue(DockerAvailability.isAvailable(),
+ "Docker daemon must be available for Firehose Redshift delivery integration tests");
+ }
+
+ @AfterEach
+ void cleanUp() {
+ if (clusterId != null) {
+ redshiftService.deleteCluster(clusterId);
+ }
+ }
+
+ private RedshiftDestination destination(String clusterName, String table, String stageBucket) throws Exception {
+ return mapper.readValue("""
+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",
+ "ClusterJDBCURL": "jdbc:redshift://%s.abc123.us-east-1.redshift.amazonaws.com:5439/dev",
+ "Username": "admin", "Password": "Secret123",
+ "CopyCommand": {"DataTableName": "%s", "CopyOptions": "json 'auto'"},
+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose",
+ "BucketARN": "arn:aws:s3:::%s"}}
+ """.formatted(clusterName, table, stageBucket), RedshiftDestination.class);
+ }
+
+ @Test
+ void putRecordIsLoadedIntoTheRedshiftTable() throws Exception {
+ clusterId = "fh-rs-e2e";
+ Cluster cluster = redshiftService.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+ try (Connection connection = RedshiftConnections.waitForConnection(cluster); Statement statement = connection.createStatement()) {
+ statement.execute("CREATE TABLE fh_events (id int, name text)");
+ }
+ firehoseService.createDeliveryStream("us-east-1", "fh-to-redshift", null,
+ destination(clusterId, "fh_events", "fh-rs-stage"), List.of(), null, null);
+ firehoseService.putRecord("fh-to-redshift",
+ new Record("{\"id\":1,\"name\":\"alice\"}".getBytes(StandardCharsets.UTF_8)));
+
+ firehoseService.flush("fh-to-redshift");
+
+ Awaitility.await()
+ .atMost(Duration.ofSeconds(10))
+ .pollInterval(Duration.ofMillis(200))
+ .untilAsserted(() -> {
+ try (Connection connection = RedshiftConnections.waitForConnection(cluster);
+ Statement statement = connection.createStatement();
+ ResultSet rows = statement.executeQuery("SELECT id, name FROM fh_events")) {
+ assertTrue(rows.next(), "expected the delivered row in fh_events");
+ assertEquals(1, rows.getInt("id"));
+ assertEquals("alice", rows.getString("name"));
+ assertFalse(rows.next());
+ }
+ });
+ }
+
+ @Test
+ void missingTableKeepsTheStagedObjectAndTheRecord() throws Exception {
+ clusterId = "fh-rs-no-table";
+ redshiftService.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+ firehoseService.createDeliveryStream("us-east-1", "fh-no-table", null,
+ destination(clusterId, "does_not_exist", "fh-rs-no-table-stage"), List.of(), null, null);
+ firehoseService.putRecord("fh-no-table",
+ new Record("{\"id\":1,\"name\":\"alice\"}".getBytes(StandardCharsets.UTF_8)));
+
+ firehoseService.flush("fh-no-table");
+
+ String listing = given().when().get("/fh-rs-no-table-stage").then().statusCode(200).extract().asString();
+ assertTrue(listing.contains("<Key>"), "expected the staged object, got: " + listing);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryTest.java
new file mode 100644
index 0000000000..a23e438304
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDeliveryTest.java
@@ -0,0 +1,170 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;
+import io.github.hectorvent.floci.services.redshift.RedshiftService;
+import io.github.hectorvent.floci.services.redshift.model.Cluster;
+import io.github.hectorvent.floci.services.redshift.model.Endpoint;
+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
+import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class FirehoseRedshiftDeliveryTest {
+
+ private final ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();
+ private RedshiftService redshift;
+ private SecretsManagerService secrets;
+ private final List<String[]> executed = new ArrayList<>();
+ private FirehoseRedshiftDelivery delivery;
+
+ @BeforeEach
+ void setUp() {
+ redshift = mock(RedshiftService.class);
+ secrets = mock(SecretsManagerService.class);
+ Cluster cluster = mock(Cluster.class);
+ when(cluster.getEndpoint()).thenReturn(new Endpoint("host.docker.internal", 41234));
+ when(cluster.getMasterUsername()).thenReturn("admin");
+ when(redshift.describeClustersForAccount("000000000000", "c1")).thenReturn(List.of(cluster));
+ delivery = new FirehoseRedshiftDelivery(redshift, secrets, mapper,
+ (url, user, password, sql) -> executed.add(new String[] {url, user, password, sql}));
+ }
+
+ private RedshiftDestination destination() throws Exception {
+ return mapper.readValue("""
+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",
+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",
+ "Username": "admin", "Password": "Secret123",
+ "CopyCommand": {"DataTableName": "events", "CopyOptions": "json 'auto'"}}
+ """, RedshiftDestination.class);
+ }
+
+ @Test
+ void runsCopyWithoutAnIamRoleThroughTheProxyPortInSimpleMode() throws Exception {
+ delivery.copy("000000000000", "us-east-1", destination(), "stage", "k1");
+
+ assertEquals(1, executed.size());
+ assertEquals("jdbc:postgresql://127.0.0.1:41234/dev?preferQueryMode=simple", executed.get(0)[0]);
+ assertEquals("admin", executed.get(0)[1]);
+ assertEquals("Secret123", executed.get(0)[2]);
+ assertEquals("COPY events FROM 's3://stage/k1' json 'auto'", executed.get(0)[3]);
+ }
+
+ @Test
+ void secretOverridesInlineCredentialsWhenEnabled() throws Exception {
+ RedshiftDestination dest = destination();
+ RedshiftDestination.SecretsManagerConfiguration cfg = new RedshiftDestination.SecretsManagerConfiguration();
+ cfg.setEnabled(true);
+ cfg.setSecretArn("arn:aws:secretsmanager:us-east-1:000000000000:secret:rs-creds-AbCdEf");
+ dest.setSecretsManagerConfiguration(cfg);
+ SecretVersion version = mock(SecretVersion.class);
+ when(version.getSecretString()).thenReturn("{\"username\":\"fromsecret\",\"password\":\"pw-secret\"}");
+ when(secrets.getSecretValue(eq(cfg.getSecretArn()), any(), any(), eq("us-east-1"))).thenReturn(version);
+
+ delivery.copy("000000000000", "us-east-1", dest, "stage", "k1");
+
+ assertEquals("fromsecret", executed.get(0)[1]);
+ assertEquals("pw-secret", executed.get(0)[2]);
+ }
+
+ @Test
+ void disabledSecretsManagerKeepsTheInlineCredentials() throws Exception {
+ RedshiftDestination dest = destination();
+ RedshiftDestination.SecretsManagerConfiguration cfg = new RedshiftDestination.SecretsManagerConfiguration();
+ cfg.setEnabled(false);
+ dest.setSecretsManagerConfiguration(cfg);
+
+ delivery.copy("000000000000", "us-east-1", dest, "stage", "k1");
+
+ assertEquals("Secret123", executed.get(0)[2]);
+ }
+
+ @Test
+ void unknownClusterSurfacesAsAwsException() {
+ when(redshift.describeClustersForAccount("000000000000", "c1"))
+ .thenThrow(new AwsException("ClusterNotFound", "Cluster c1 not found", 404));
+
+ assertThrows(AwsException.class,
+ () -> delivery.copy("000000000000", "us-east-1", destination(), "stage", "k"));
+ }
+
+ @Test
+ void jdbcFailureIsWrappedWithTableClusterAndSqlState() {
+ FirehoseRedshiftDelivery failing = new FirehoseRedshiftDelivery(redshift, secrets, mapper,
+ (url, user, password, sql) -> {
+ throw new SQLException("relation \"events\" does not exist", "42P01");
+ });
+
+ IllegalStateException e = assertThrows(IllegalStateException.class,
+ () -> failing.copy("000000000000", "us-east-1", destination(), "stage", "k"));
+
+ assertTrue(e.getMessage().contains("events"), e.getMessage());
+ assertTrue(e.getMessage().contains("c1"), e.getMessage());
+ assertTrue(e.getMessage().contains("42P01"), e.getMessage());
+ }
+
+ @Test
+ void jdbcConnectionsCarryConnectAndSocketTimeouts() {
+ Properties properties = FirehoseRedshiftDelivery.connectionProperties("admin", "Secret123");
+
+ assertEquals("admin", properties.getProperty("user"));
+ assertEquals("Secret123", properties.getProperty("password"));
+ assertTrue(Integer.parseInt(properties.getProperty("connectTimeout")) > 0);
+ assertTrue(Integer.parseInt(properties.getProperty("socketTimeout")) > 0);
+ }
+
+ @Test
+ void closeFailureAfterACommittedCopyIsNotAFailedDelivery() throws Exception {
+ Connection connection = mock(Connection.class);
+ Statement statement = mock(Statement.class);
+ when(connection.createStatement()).thenReturn(statement);
+ doThrow(new SQLException("connection reset")).when(connection).close();
+
+ FirehoseRedshiftDelivery.executeAndClose(connection, "COPY t FROM 's3://b/k'");
+
+ verify(statement).execute("COPY t FROM 's3://b/k'");
+ }
+
+ @Test
+ void failedCopyIsReportedAndTheConnectionStillClosed() throws Exception {
+ Connection connection = mock(Connection.class);
+ Statement statement = mock(Statement.class);
+ when(connection.createStatement()).thenReturn(statement);
+ when(statement.execute(any())).thenThrow(new SQLException("relation does not exist", "42P01"));
+
+ SQLException e = assertThrows(SQLException.class,
+ () -> FirehoseRedshiftDelivery.executeAndClose(connection, "COPY t FROM 's3://b/k'"));
+
+ assertEquals("42P01", e.getSQLState());
+ verify(connection).close();
+ }
+
+ @Test
+ void destinationWithoutAnyPasswordFailsWithAClearMessage() throws Exception {
+ RedshiftDestination dest = destination();
+ dest.setPassword(null);
+
+ IllegalStateException e = assertThrows(IllegalStateException.class,
+ () -> delivery.copy("000000000000", "us-east-1", dest, "stage", "k"));
+
+ assertTrue(e.getMessage().contains("Password"), e.getMessage());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDestinationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDestinationIntegrationTest.java
index 31f732df47..4cdf14d8f1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDestinationIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftDestinationIntegrationTest.java
@@ -233,7 +233,7 @@ class FirehoseRedshiftDestinationIntegrationTest {
}
@Test
- void flushStagesTheBatchInTheIntermediateBucket() {
+ void flushWithAMissingClusterKeepsTheStagedObject() {
post("CreateDeliveryStream", createBody("rs-flush", "rs-flush-stage")).statusCode(200);
String data = Base64.getEncoder().encodeToString("{\"id\":1}".getBytes(StandardCharsets.UTF_8));
post("PutRecord", "{\"DeliveryStreamName\": \"rs-flush\", \"Record\": {\"Data\": \"" + data + "\"}}")
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java
new file mode 100644
index 0000000000..b21dd10bf6
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseRedshiftServiceTest.java
@@ -0,0 +1,279 @@
+package io.github.hectorvent.floci.services.firehose;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.firehose.model.Record;
+import io.github.hectorvent.floci.services.firehose.model.RedshiftDestination;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class FirehoseRedshiftServiceTest {
+
+ private static final String STREAM = "rs-delivery";
+ private static final String BACKUP_ENABLED = ", \"S3BackupMode\": \"Enabled\", \"S3BackupConfiguration\": "
+ + "{\"RoleARN\": \"arn:aws:iam::000000000000:role/firehose\", \"BucketARN\": \"arn:aws:s3:::bk\"}";
+
+ private final ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();
+ private FirehoseService service;
+ private S3Service s3Service;
+ private FirehoseRedshiftDelivery delivery;
+ private MutableClock clock;
+ private StorageFactory storageFactory;
+ private RegionResolver regionResolver;
+ private EmulatorConfig config;
+
+ @BeforeEach
+ void setUp() {
+ storageFactory = Mockito.mock(StorageFactory.class);
+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();
+ when(storageFactory.create(anyString(), anyString(), any()))
+ .thenAnswer(invocation -> backends.computeIfAbsent(
+ invocation.getArgument(0) + "/" + invocation.getArgument(1),
+ k -> AccountAwareStorageBackend.inMemory("000000000000")));
+ s3Service = Mockito.mock(S3Service.class);
+ delivery = Mockito.mock(FirehoseRedshiftDelivery.class);
+ clock = new MutableClock();
+
+ EmulatorConfig.FirehoseServiceConfig firehoseCfg = mock(EmulatorConfig.FirehoseServiceConfig.class);
+ when(firehoseCfg.enabled()).thenReturn(true);
+ when(firehoseCfg.tickIntervalSeconds()).thenReturn(10L);
+ when(firehoseCfg.flushRecordCount()).thenReturn(0);
+ EmulatorConfig.ServicesConfig servicesCfg = mock(EmulatorConfig.ServicesConfig.class);
+ when(servicesCfg.firehose()).thenReturn(firehoseCfg);
+ config = mock(EmulatorConfig.class);
+ when(config.services()).thenReturn(servicesCfg);
+ regionResolver = new RegionResolver("us-east-1", "000000000000");
+
+ service = new FirehoseService(storageFactory, s3Service, new KinesisService(storageFactory, regionResolver),
+ regionResolver, clock, config, mock(FirehoseParquetConverter.class),
+ mock(FirehoseLambdaTransformer.class), delivery, Runnable::run);
+ }
+
+ private void createStream(String extraMembers) throws Exception {
+ RedshiftDestination destination = mapper.readValue("""
+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",
+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",
+ "Username": "admin", "Password": "Secret123",
+ "CopyCommand": {"DataTableName": "events"},
+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}
+ %s}
+ """.formatted(extraMembers), RedshiftDestination.class);
+ service.createDeliveryStream("us-east-1", STREAM, null, destination, List.of(), null, null);
+ service.putRecord(STREAM, new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));
+ }
+
+ private void copyFails() {
+ doThrow(new IllegalStateException("relation does not exist"))
+ .when(delivery).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ private void verifyStagedTimes(int times) {
+ verify(s3Service, times(times)).putObject(eq("stage"), anyString(), any(byte[].class), anyString(),
+ anyMap(), any());
+ }
+
+ @Test
+ void successfulCopyConsumesTheBatchOnce() throws Exception {
+ createStream("");
+
+ service.flush(STREAM);
+ service.flush(STREAM);
+
+ verify(delivery, times(1)).copy(eq("000000000000"), eq("us-east-1"), any(), eq("stage"), anyString());
+ verifyStagedTimes(1);
+ }
+
+ @Test
+ void failedCopyIsLoggedAndNotRestagedOrRebuffered() throws Exception {
+ createStream("");
+ copyFails();
+
+ service.flush(STREAM);
+ service.flush(STREAM);
+
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), eq("stage"), anyString());
+ verifyStagedTimes(1);
+ }
+
+ @Test
+ void deliveryUnavailableLeavesStagedObjectInS3() throws Exception {
+ FirehoseService serviceWithoutDelivery = new FirehoseService(storageFactory, s3Service,
+ new KinesisService(storageFactory, regionResolver), regionResolver, clock, config,
+ mock(FirehoseParquetConverter.class), mock(FirehoseLambdaTransformer.class), null, Runnable::run);
+ RedshiftDestination destination = mapper.readValue("""
+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",
+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",
+ "Username": "admin", "Password": "Secret123",
+ "CopyCommand": {"DataTableName": "events"},
+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}}
+ """, RedshiftDestination.class);
+ serviceWithoutDelivery.createDeliveryStream("us-east-1", "fh-no-delivery", null, destination, List.of(), null, null);
+ serviceWithoutDelivery.putRecord("fh-no-delivery", new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));
+
+ serviceWithoutDelivery.flush("fh-no-delivery");
+
+ verifyStagedTimes(1);
+ verify(delivery, never()).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void backupEnabledWritesTheBatchToTheBackupBucket() throws Exception {
+ createStream(BACKUP_ENABLED);
+
+ service.flush(STREAM);
+
+ verifyStagedTimes(1);
+ verify(s3Service, times(1)).putObject(eq("bk"), anyString(), any(byte[].class), anyString(), anyMap(), any());
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), eq("stage"), anyString());
+ }
+
+ @Test
+ void backupObjectUsesTheBackupConfigurationPrefix() throws Exception {
+ createStream(", \"S3BackupMode\": \"Enabled\", \"S3BackupConfiguration\": "
+ + "{\"RoleARN\": \"arn:aws:iam::000000000000:role/firehose\", \"BucketARN\": \"arn:aws:s3:::bk\","
+ + " \"Prefix\": \"backup/\"}");
+
+ service.flush(STREAM);
+
+ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);
+ verify(s3Service).putObject(eq("bk"), keys.capture(), any(byte[].class), anyString(), anyMap(), any());
+ assertTrue(keys.getValue().startsWith("backup/"), keys.getValue());
+ }
+
+ @Test
+ void aFailingBackupWriteStillLoadsTheStagedBatchAndDoesNotRestageIt() throws Exception {
+ createStream(BACKUP_ENABLED);
+ doThrow(new IllegalStateException("backup bucket unavailable")).when(s3Service)
+ .putObject(eq("bk"), anyString(), any(byte[].class), anyString(), anyMap(), any());
+
+ service.flush(STREAM);
+ service.flush(STREAM);
+
+ verifyStagedTimes(1);
+ verify(delivery, times(1)).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void aFailingPrimaryWriteWritesNoBackupSoNothingIsDelivered() throws Exception {
+ createStream(BACKUP_ENABLED);
+ doThrow(new IllegalStateException("s3 unavailable")).when(s3Service)
+ .putObject(eq("stage"), anyString(), any(byte[].class), anyString(), anyMap(), any());
+
+ service.flush(STREAM);
+
+ verify(s3Service, never()).putObject(eq("bk"), anyString(), any(byte[].class), anyString(), anyMap(), any());
+ verify(delivery, never()).copy(anyString(), anyString(), any(), anyString(), anyString());
+ }
+
+ @Test
+ void rejectedExecutionDoesNotRestoreRecordsToBuffer() throws Exception {
+ Executor rejectingExecutor = r -> {
+ throw new RejectedExecutionException("executor shut down");
+ };
+ FirehoseService rejectingService = new FirehoseService(storageFactory, s3Service,
+ new KinesisService(storageFactory, regionResolver), regionResolver, clock, config,
+ mock(FirehoseParquetConverter.class), mock(FirehoseLambdaTransformer.class), delivery, rejectingExecutor);
+
+ RedshiftDestination destination = mapper.readValue("""
+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",
+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",
+ "Username": "admin", "Password": "Secret123",
+ "CopyCommand": {"DataTableName": "events"},
+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}}
+ """, RedshiftDestination.class);
+ rejectingService.createDeliveryStream("us-east-1", "fh-rejected", null, destination, List.of(), null, null);
+ rejectingService.putRecord("fh-rejected", new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));
+
+ rejectingService.flush("fh-rejected");
+ rejectingService.flush("fh-rejected");
+
+ verify(s3Service, times(1)).putObject(eq("stage"), anyString(), any(byte[].class), anyString(),
+ anyMap(), any());
+ }
+
+ @Test
+ void onPreShutdownFlushesBuffersAndRunsCopyBeforeShuttingDown() throws Exception {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ FirehoseService shutdownService = new FirehoseService(storageFactory, s3Service,
+ new KinesisService(storageFactory, regionResolver), regionResolver, clock, config,
+ mock(FirehoseParquetConverter.class), mock(FirehoseLambdaTransformer.class), delivery, executor);
+
+ RedshiftDestination destination = mapper.readValue("""
+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",
+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",
+ "Username": "admin", "Password": "Secret123",
+ "CopyCommand": {"DataTableName": "events"},
+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}}
+ """, RedshiftDestination.class);
+ shutdownService.createDeliveryStream("us-east-1", "fh-shutdown", null, destination, List.of(), null, null);
+ shutdownService.putRecord("fh-shutdown", new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));
+
+ shutdownService.onPreShutdown(null);
+
+ verify(s3Service, times(1)).putObject(eq("stage"), anyString(), any(byte[].class), anyString(),
+ anyMap(), any());
+ verify(delivery, times(1)).copy(eq("000000000000"), eq("us-east-1"), any(), eq("stage"), anyString());
+ assertTrue(executor.isTerminated());
+ }
+
+ @Test
+ void onPreShutdownDropsQueuedTasksWhenTerminationTimesOut() throws Exception {
+ ExecutorService mockExecutor = mock(ExecutorService.class);
+ FirehoseService shutdownService = new FirehoseService(storageFactory, s3Service,
+ new KinesisService(storageFactory, regionResolver), regionResolver, clock, config,
+ mock(FirehoseParquetConverter.class), mock(FirehoseLambdaTransformer.class), delivery, mockExecutor);
+
+ RedshiftDestination destination = mapper.readValue("""
+ {"RoleARN": "arn:aws:iam::000000000000:role/firehose",
+ "ClusterJDBCURL": "jdbc:redshift://c1.abc.us-east-1.redshift.amazonaws.com:5439/dev",
+ "Username": "admin", "Password": "Secret123",
+ "CopyCommand": {"DataTableName": "events"},
+ "S3Configuration": {"RoleARN": "arn:aws:iam::000000000000:role/firehose", "BucketARN": "arn:aws:s3:::stage"}}
+ """, RedshiftDestination.class);
+ shutdownService.createDeliveryStream("us-east-1", "fh-timeout", null, destination, List.of(), null, null);
+ shutdownService.putRecord("fh-timeout", new Record("{\"id\":1}".getBytes(StandardCharsets.UTF_8)));
+
+ when(mockExecutor.awaitTermination(anyLong(), any())).thenReturn(false);
+ when(mockExecutor.shutdownNow()).thenReturn(List.of(() -> {}));
+
+ shutdownService.onPreShutdown(null);
+
+ InOrder inOrder = Mockito.inOrder(mockExecutor);
+ inOrder.verify(mockExecutor).execute(any());
+ inOrder.verify(mockExecutor).shutdown();
+ inOrder.verify(mockExecutor).awaitTermination(5, TimeUnit.SECONDS);
+ inOrder.verify(mockExecutor).shutdownNow();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySqlTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySqlTest.java
index 8f0a7e960c..9ff89dea34 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySqlTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/RedshiftCopySqlTest.java
@@ -4,6 +4,7 @@ import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class RedshiftCopySqlTest {
@@ -118,6 +119,50 @@ class RedshiftCopySqlTest {
assertTrue(RedshiftCopySql.hasStatementSeparator("NULL AS $x"));
}
+ @Test
+ void buildsCopyWithOptionsAndNoIamRole() {
+ String sql = RedshiftCopySql.build("events", null, "stage", "2026/10/19/f.gz", "json 'auto' gzip");
+
+ assertEquals("COPY events FROM 's3://stage/2026/10/19/f.gz' json 'auto' gzip", sql);
+ }
+
+ @Test
+ void buildDropsTheClosingTerminatorOfTheApiReferenceExample() {
+ String sql = RedshiftCopySql.build("t", null, "b", "k", "delimiter '\\t' lzop;");
+
+ assertEquals("COPY t FROM 's3://b/k' delimiter '\\t' lzop", sql);
+ }
+
+ @Test
+ void buildKeepsATerminatorInsideAQuotedValue() {
+ String sql = RedshiftCopySql.build("t", null, "b", "k", "DELIMITER ';' CSV");
+
+ assertEquals("COPY t FROM 's3://b/k' DELIMITER ';' CSV", sql);
+ }
+
+ @Test
+ void tableAndColumnsAreKeptVerbatimSoRedshiftCaseFoldingApplies() {
+ String sql = RedshiftCopySql.build("Public.MyEvents", "Id, \"Full Name\"", "stage", "k", null);
+
+ assertEquals("COPY Public.MyEvents (Id, \"Full Name\") FROM 's3://stage/k'", sql);
+ }
+
+ @Test
+ void buildRejectsNamesAndOptionsThatCouldCarryAnotherStatement() {
+ assertThrows(IllegalArgumentException.class, () -> RedshiftCopySql.build("t; DROP TABLE x", null, "b", "k", null));
+ assertThrows(IllegalArgumentException.class,
+ () -> RedshiftCopySql.build("t", "id, name); DROP TABLE t; --", "b", "k", null));
+ assertThrows(IllegalArgumentException.class, () -> RedshiftCopySql.build("t", null, "b", "k", "gzip; DROP TABLE t"));
+ }
+
+ @Test
+ void objectKeyThatTheProxyCouldNotParseIsRejected() {
+ for (String key : new String[] {"my prefix/f.gz", "it's/f.gz", "say\"/f.gz"}) {
+ assertThrows(IllegalArgumentException.class,
+ () -> RedshiftCopySql.build("t", null, "stage", key, null), key);
+ }
+ }
+
@Test
void dollarQuotingAndCommentsAreRefused() {
assertTrue(RedshiftCopySql.hasStatementSeparator("NULL AS $$'$$ gzip"));
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftConnections.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftConnections.java
new file mode 100644
index 0000000000..55c5eade00
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftConnections.java
@@ -0,0 +1,45 @@
+package io.github.hectorvent.floci.services.redshift;
+
+import io.github.hectorvent.floci.services.redshift.model.Cluster;
+import org.awaitility.Awaitility;
+import org.awaitility.core.ConditionTimeoutException;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.time.Duration;
+import java.util.Objects;
+
+public final class RedshiftConnections {
+
+ private RedshiftConnections() {}
+
+ public static String jdbcUrl(Cluster cluster) {
+ return "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort() + "/dev";
+ }
+
+ public static Connection waitForConnection(Cluster cluster) throws SQLException {
+ return waitForConnection(cluster, "admin", "Secret123");
+ }
+
+ public static Connection waitForConnection(Cluster cluster, String username, String password) throws SQLException {
+ return waitForConnection(jdbcUrl(cluster), username, password);
+ }
+
+ public static Connection waitForConnection(String host, int port, String username, String password) throws SQLException {
+ return waitForConnection("jdbc:postgresql://" + host + ":" + port + "/dev", username, password);
+ }
+
+ public static Connection waitForConnection(String jdbcUrl, String username, String password) throws SQLException {
+ try {
+ return Awaitility.await()
+ .atMost(Duration.ofSeconds(30))
+ .pollDelay(Duration.ZERO)
+ .pollInterval(Duration.ofMillis(500))
+ .ignoreExceptions()
+ .until(() -> DriverManager.getConnection(jdbcUrl, username, password), Objects::nonNull);
+ } catch (ConditionTimeoutException e) {
+ return DriverManager.getConnection(jdbcUrl, username, password);
+ }
+ }
+}