Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(redshift): emulate Redshift UNLOAD TO s3 over Simple Query protocol (#3129)

Vũ Thế Huy4570bf62c5f8
diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java
index 28fa758ee9..db816e09e1 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java
@@ -13,6 +13,16 @@ import software.amazon.awssdk.services.redshift.model.DescribeClustersRequest;
import software.amazon.awssdk.services.redshift.model.DescribeClustersResponse;
import software.amazon.awssdk.services.redshift.model.DeleteClusterRequest;
import software.amazon.awssdk.services.redshift.model.DeleteClusterResponse;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -59,6 +69,45 @@ public class RedshiftTest {
@Test
@Order(2)
+ public void testUnloadOverSimpleQueryWritesToS3() throws Exception {
+ RedshiftClient client = getClient();
+ Cluster cluster = client.describeClusters(DescribeClustersRequest.builder()
+ .clusterIdentifier("test-cluster")
+ .build()).clusters().get(0);
+ String jdbcUrl = "jdbc:postgresql://" + cluster.endpoint().address() + ":"
+ + cluster.endpoint().port() + "/dev?preferQueryMode=simple";
+
+ S3Client s3 = TestFixtures.s3Client();
+ String bucket = "redshift-unload-compat";
+ s3.createBucket(CreateBucketRequest.builder().bucket(bucket).build());
+
+ try (Connection conn = DriverManager.getConnection(jdbcUrl, "admin", "Password123");
+ Statement st = conn.createStatement()) {
+ st.execute("DROP TABLE IF EXISTS unload_src");
+ st.execute("CREATE TABLE unload_src (id int, name text)");
+ st.execute("INSERT INTO unload_src VALUES (1, 'alice'), (2, 'bob')");
+ st.execute("UNLOAD ('select id, name from unload_src order by id') "
+ + "TO 's3://redshift-unload-compat/out/'");
+ }
+
+ ListObjectsV2Response listing = s3.listObjectsV2(ListObjectsV2Request.builder()
+ .bucket(bucket)
+ .prefix("out/")
+ .build());
+ assertTrue(listing.keyCount() >= 1, "UNLOAD must write at least one object");
+
+ StringBuilder all = new StringBuilder();
+ listing.contents().stream()
+ .sorted((a, b) -> a.key().compareTo(b.key()))
+ .forEach(obj -> all.append(new String(
+ s3.getObjectAsBytes(GetObjectRequest.builder().bucket(bucket).key(obj.key()).build())
+ .asByteArray(),
+ StandardCharsets.UTF_8)));
+ assertEquals("1|alice\n2|bob\n", all.toString());
+ }
+
+ @Test
+ @Order(3)
public void testDeleteCluster() {
RedshiftClient client = getClient();
DeleteClusterResponse res = client.deleteCluster(DeleteClusterRequest.builder()
diff --git a/docs/services/redshift.md b/docs/services/redshift.md
index 8d632a0085..958877cff3 100644
--- a/docs/services/redshift.md
+++ b/docs/services/redshift.md
@@ -175,11 +175,45 @@ order) through its own S3 service and streams the rows into the backing PostgreS
- The rewrite is textual (regex-based). It masks single-quoted string literals first, so `DEFAULT` / `CHECK` string values are safe, but it is **not** comment-aware and does not recognize escape strings (`E'...'`): an apostrophe inside a `--` or `/* */` comment can make the rewrite skip a Redshift clause. That fails safe: the statement then reaches PostgreSQL, which returns its own syntax error, but avoid apostrophes-in-comments in `CREATE TABLE` / `ALTER TABLE`.
- A `rewrite` failure or any statement the interceptor does not recognize is forwarded unmodified (fail-open); PostgreSQL then rejects the Redshift-only syntax itself.
- Simple Query ('Q') messages larger than 16 MiB bypass the interceptor and stream through verbatim without heap buffering; non-query traffic also streams through with no size limit.
-- `UNLOAD (...) TO 's3://...'` is **not** yet emulated (planned).
+
+### UNLOAD to S3
+
+`UNLOAD ('<select-statement>') TO 's3://<bucket>/<prefix>' [options]` sent over the
+Simple Query protocol runs the select on the backing PostgreSQL container and writes
+the result to S3 as one or more objects under `<prefix>`.
+
+- Framing defaults to pipe-delimited text; `FORMAT CSV` (or `CSV`) switches to CSV
+ with a `,` default. `DELIMITER`, `HEADER`, `NULL AS`, and `ADDQUOTES` are honoured
+ (`ADDQUOTES` and `HEADER` force CSV framing on PostgreSQL 15).
+- `PARALLEL ON` (default) names objects `<prefix>0000_part_00`,
+ `<prefix>0001_part_00`, and so on; `PARALLEL OFF` names them `<prefix>000`,
+ `<prefix>001`, and so on. The emulator has a single backend node, so more than one
+ object appears only when the result exceeds the per-file size. When `HEADER` is
+ set, the header row is repeated at the top of every object.
+- `MAXFILESIZE [AS] <n> [MB|GB]` sets the per-file size; a bare number is bytes.
+ Each file is buffered in memory, so the default is 6 MiB (real Redshift defaults
+ to 6.2 GB) and a whole UNLOAD result is capped at 256 MiB; a larger result fails
+ with a SQL error (54000). A `MAXFILESIZE` above that 256 MiB cap is not
+ intercepted at all: the statement is forwarded and PostgreSQL reports its own error.
+- A zero-row result still writes one object (empty, or the header row alone when
+ `HEADER` is set).
+- `GZIP` compresses each object and appends `.gz` to its key.
+- `MANIFEST` writes `<prefix>manifest` listing every object with its
+ `content_length`.
+- Without `ALLOWOVERWRITE`, a non-empty target prefix fails with SQL error XX000 and the select
+ does not run. A failed UNLOAD then removes any objects it had already written. With
+ `ALLOWOVERWRITE` a failed UNLOAD leaves its objects in place (they may have replaced prior data,
+ so they are not deleted); a `MANIFEST` request that fails this way can leave data objects without
+ a manifest, and rerunning the same statement overwrites them.
+- S3 access is authorized as an unsigned request, like COPY from S3.
+- Any other option (`PARQUET`, `ENCRYPTED`, `REGION`, `IAM_ROLE` / `CREDENTIALS`,
+ `ZSTD`, `EXTENSION`, `CLEANPATH`, `PARTITION`, and so on) is not intercepted; the
+ statement is forwarded and PostgreSQL reports its own error.
+- Extended Query protocol UNLOAD (a JDBC `PreparedStatement`) is not intercepted.
## Out of Scope
-- Real Redshift SQL semantics: the data plane is stock PostgreSQL. Redshift-only table DDL keywords (DISTSTYLE / DISTKEY / SORTKEY / ENCODE) are stripped so CREATE TABLE / ALTER TABLE executes (see [SQL Interceptor](#sql-interceptor)), but the distribution/sort behavior they request is not; UNLOAD to S3 and SUPER/SPECTRUM are not emulated.
+- Real Redshift SQL semantics: the data plane is stock PostgreSQL. Redshift-only table DDL keywords (DISTSTYLE / DISTKEY / SORTKEY / ENCODE) are stripped so CREATE TABLE / ALTER TABLE executes (see [SQL Interceptor](#sql-interceptor)), but the distribution/sort behavior they request is not; SUPER/SPECTRUM are not emulated.
- Multi-node clusters: `NodeType` and `NumberOfNodes` are stored as metadata; every cluster is a single PostgreSQL container.
- Parameter groups apply no real engine settings; values are stored and echoed back only.
- Subnet groups, VPC routing, and security groups are metadata only.
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java
index 215d927975..c051efd71c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java
@@ -406,7 +406,7 @@ public class RedshiftContainerManager {
lastOutput = e.getMessage();
}
try {
- java.util.concurrent.TimeUnit.SECONDS.sleep(1);
+ TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Interrupted initializing " + description + " in " + containerName, e);
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java
index 3c990300f1..1fe4cb6eeb 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java
@@ -6,12 +6,16 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
- * Recognises a Simple Query statement that is an S3 {@code COPY <table> FROM 's3://...'}.
- * Any other statement, or a COPY that uses an option this simulator cannot honour, returns
- * {@code null} so the bridge falls back to DDL rewriting and PostgreSQL reports its own error.
+ * Recognises a Simple Query statement that is an S3 {@code COPY <table> FROM 's3://...'}
+ * or {@code UNLOAD ('<select>') TO 's3://...'}. Any other statement, or a statement that uses
+ * an option this simulator cannot honour, returns {@code null} so the bridge falls back to DDL
+ * rewriting and PostgreSQL reports its own error.
*/
public final class CopyStatementParser {
+ public sealed interface S3Statement permits S3CopyFrom, S3Unload {
+ }
+
public record S3CopyFrom(
String targetTable,
List<String> columns,
@@ -21,13 +25,52 @@ public final class CopyStatementParser {
int headerLines,
boolean gzip,
boolean csv,
- String nullAs) {
+ String nullAs) implements S3Statement {
+ }
+
+ public record S3Unload(
+ String selectQuery,
+ String bucket,
+ String prefix,
+ String delimiter,
+ boolean header,
+ boolean gzip,
+ boolean csv,
+ boolean addQuotes,
+ String nullAs,
+ boolean manifest,
+ boolean allowOverwrite,
+ boolean parallel,
+ long maxFileSizeBytes) implements S3Statement {
}
private static final Pattern COPY_PATTERN = Pattern.compile(
"(?is)^\\s*COPY\\s+((?:\"[^\"]*\"|[^\\s(])+)\\s*(?:\\(([^)]*)\\))?\\s+FROM\\s*"
+ "['\"]s3://([^/'\"\\s]+)(?:/([^'\"\\s]*))?['\"]\\s*(.*)$");
+ private static final Pattern UNLOAD_PATTERN = Pattern.compile(
+ "(?is)^\\s*UNLOAD\\s*\\(\\s*'(.*)'\\s*\\)\\s*TO\\s*"
+ + "['\"]s3://([^/'\"\\s]+)(?:/([^'\"\\s]*))?['\"]\\s*(.*)$");
+
+ private static final Pattern PARALLEL_PATTERN = Pattern.compile(
+ "(?i)\\bPARALLEL\\b(?:\\s+(ON|OFF|TRUE|FALSE))?");
+ private static final Pattern MAXFILESIZE_PATTERN = Pattern.compile(
+ "(?i)\\bMAXFILESIZE\\s+(?:AS\\s+)?(\\d+(?:\\.\\d+)?)\\s*(MB|GB)?\\b");
+ private static final Pattern ADDQUOTES_PATTERN = Pattern.compile("(?i)\\bADDQUOTES\\b");
+ private static final Pattern MANIFEST_PATTERN = Pattern.compile("(?i)\\bMANIFEST\\b");
+ private static final Pattern ALLOWOVERWRITE_PATTERN = Pattern.compile("(?i)\\bALLOWOVERWRITE\\b");
+
+ /**
+ * UNLOAD options this simulator cannot honour. Distinct from the COPY set:
+ * MANIFEST / ALLOWOVERWRITE / PARALLEL / MAXFILESIZE are UNLOAD-supported here,
+ * while EXTENSION / CLEANPATH / PARTITION are UNLOAD-only and unsupported.
+ */
+ private static final Pattern UNLOAD_UNSUPPORTED_CLAUSE = Pattern.compile(
+ "(?i)\\b(FIXEDWIDTH|PARQUET|AVRO|ORC|JSON|SHAPEFILE|BZIP2|LZOP|ZSTD"
+ + "|ENCRYPTED|ENCODING|REGION|CREDENTIALS|IAM_ROLE|ACCESS_KEY_ID"
+ + "|SECRET_ACCESS_KEY|SESSION_TOKEN|MASTER_SYMMETRIC_KEY|KMS_KEY_ID"
+ + "|EXTENSION|CLEANPATH|PARTITION|MAXFILESIZE\\s+\\d+\\s*(?:TB|PB))\\b");
+
private static final Pattern QUALIFIED_NAME = Pattern.compile(
"(?:[A-Za-z_][A-Za-z0-9_$]*|\"[^\"]+\")(?:\\.(?:[A-Za-z_][A-Za-z0-9_$]*|\"[^\"]+\"))?");
private static final Pattern SIMPLE_NAME = Pattern.compile("[A-Za-z_][A-Za-z0-9_$]*|\"[^\"]+\"");
@@ -61,11 +104,174 @@ public final class CopyStatementParser {
private CopyStatementParser() {
}
- public static S3CopyFrom parse(String sql) {
+ public static S3Statement parse(String sql) {
if (sql == null || sql.isBlank()) {
return null;
}
String cleaned = stripLeadingComments(sql.trim());
+ if (cleaned.isEmpty()) {
+ return null;
+ }
+ Matcher unload = UNLOAD_PATTERN.matcher(cleaned);
+ if (unload.matches()) {
+ return parseUnload(unload);
+ }
+ return parseCopy(cleaned);
+ }
+
+ private static S3Unload parseUnload(Matcher matcher) {
+ String select = unescapeSingleQuotes(matcher.group(1).trim());
+ if (!isSafeUnloadSubquery(select)) {
+ return null;
+ }
+ String bucket = matcher.group(2);
+ String prefix = matcher.group(3) != null ? matcher.group(3) : "";
+ String options = matcher.group(4) != null ? matcher.group(4) : "";
+
+ String flagScan = blankQuoted(options);
+ if (UNLOAD_UNSUPPORTED_CLAUSE.matcher(flagScan).find()
+ || TRAILING_STATEMENT.matcher(flagScan).find()) {
+ return null;
+ }
+ int semi = flagScan.indexOf(';');
+ if (semi >= 0) {
+ if (!flagScan.substring(semi + 1).isBlank()) {
+ return null;
+ }
+ options = options.substring(0, semi);
+ }
+
+ boolean csv = false;
+ boolean gzip = false;
+ boolean header = false;
+ boolean addQuotes = false;
+ boolean manifest = false;
+ boolean allowOverwrite = false;
+ boolean parallel = true;
+ String nullAs = null;
+ String delimiter = null;
+ long maxFileSizeBytes = 0L;
+
+ boolean seenCsv = false;
+ boolean seenGzip = false;
+ boolean seenHeader = false;
+ boolean seenAddQuotes = false;
+ boolean seenManifest = false;
+ boolean seenAllowOverwrite = false;
+ boolean seenParallel = false;
+ boolean seenNull = false;
+ boolean seenDelimiter = false;
+ boolean seenMaxFileSize = false;
+
+ Matcher csvM = CSV_PATTERN.matcher(options);
+ Matcher gzipM = GZIP_PATTERN.matcher(options);
+ Matcher headerM = HEADER_PATTERN.matcher(options);
+ Matcher addQuotesM = ADDQUOTES_PATTERN.matcher(options);
+ Matcher manifestM = MANIFEST_PATTERN.matcher(options);
+ Matcher allowOverwriteM = ALLOWOVERWRITE_PATTERN.matcher(options);
+ Matcher parallelM = PARALLEL_PATTERN.matcher(options);
+ Matcher delimiterM = DELIMITER_PATTERN.matcher(options);
+ Matcher nullM = NULL_AS_PATTERN.matcher(options);
+ Matcher maxFileSizeM = MAXFILESIZE_PATTERN.matcher(options);
+
+ int offset = 0;
+ int len = options.length();
+ while (offset < len) {
+ while (offset < len && Character.isWhitespace(options.charAt(offset))) {
+ offset++;
+ }
+ if (offset >= len) {
+ break;
+ }
+ if (matchClause(maxFileSizeM, offset, len)) {
+ if (seenMaxFileSize) {
+ return null;
+ }
+ seenMaxFileSize = true;
+ maxFileSizeBytes = maxFileSizeToBytes(maxFileSizeM.group(1), maxFileSizeM.group(2));
+ if (maxFileSizeBytes > S3CopySimulator.UNLOAD_MAX_TOTAL_BYTES) {
+ // A per-file size the simulator cannot buffer: fail open so PostgreSQL,
+ // rather than an unrelated late "result too large" error, reports it.
+ return null;
+ }
+ offset = maxFileSizeM.end();
+ } else if (matchClause(csvM, offset, len)) {
+ if (seenCsv) {
+ return null;
+ }
+ seenCsv = true;
+ csv = true;
+ offset = csvM.end();
+ } else if (matchClause(gzipM, offset, len)) {
+ if (seenGzip) {
+ return null;
+ }
+ seenGzip = true;
+ gzip = true;
+ offset = gzipM.end();
+ } else if (matchClause(headerM, offset, len)) {
+ if (seenHeader) {
+ return null;
+ }
+ seenHeader = true;
+ header = true;
+ offset = headerM.end();
+ } else if (matchClause(addQuotesM, offset, len)) {
+ if (seenAddQuotes) {
+ return null;
+ }
+ seenAddQuotes = true;
+ addQuotes = true;
+ offset = addQuotesM.end();
+ } else if (matchClause(manifestM, offset, len)) {
+ if (seenManifest) {
+ return null;
+ }
+ seenManifest = true;
+ manifest = true;
+ offset = manifestM.end();
+ } else if (matchClause(allowOverwriteM, offset, len)) {
+ if (seenAllowOverwrite) {
+ return null;
+ }
+ seenAllowOverwrite = true;
+ allowOverwrite = true;
+ offset = allowOverwriteM.end();
+ } else if (matchClause(parallelM, offset, len)) {
+ if (seenParallel) {
+ return null;
+ }
+ seenParallel = true;
+ String v = parallelM.group(1);
+ parallel = v == null || v.equalsIgnoreCase("ON") || v.equalsIgnoreCase("TRUE");
+ offset = parallelM.end();
+ } else if (matchClause(delimiterM, offset, len)) {
+ if (seenDelimiter) {
+ return null;
+ }
+ seenDelimiter = true;
+ delimiter = extractDelimiterValue(delimiterM);
+ offset = delimiterM.end();
+ } else if (matchClause(nullM, offset, len)) {
+ if (seenNull) {
+ return null;
+ }
+ seenNull = true;
+ nullAs = extractNullValue(nullM);
+ offset = nullM.end();
+ } else {
+ return null;
+ }
+ }
+
+ if (delimiter == null) {
+ delimiter = csv ? "," : "|";
+ }
+ return new S3Unload(select, bucket, prefix, delimiter, header, gzip, csv,
+ addQuotes, nullAs, manifest, allowOverwrite, parallel, maxFileSizeBytes);
+ }
+
+ private static S3CopyFrom parseCopy(String cleaned) {
Matcher matcher = COPY_PATTERN.matcher(cleaned);
if (!matcher.matches()) {
return null;
@@ -272,4 +478,117 @@ public final class CopyStatementParser {
}
}
}
+
+ private static long maxFileSizeToBytes(String number, String unit) {
+ double value = Double.parseDouble(number);
+ long scale = 1L;
+ if (unit != null && unit.equalsIgnoreCase("MB")) {
+ scale = 1024L * 1024;
+ } else if (unit != null && unit.equalsIgnoreCase("GB")) {
+ scale = 1024L * 1024 * 1024;
+ }
+ long bytes = (long) (value * scale);
+ return bytes > 0 ? bytes : 0L;
+ }
+
+ private static String unescapeSingleQuotes(String s) {
+ return s.replace("''", "'");
+ }
+
+ /**
+ * True only if the UNLOAD subquery cannot escape the {@code COPY (<q>) TO STDOUT}
+ * wrapper it is spliced into: it must start with {@code SELECT} or {@code WITH},
+ * keep parentheses balanced (never dipping below zero), and contain no {@code ;},
+ * {@code $}, {@code --}, or {@code /*} outside a single- or double-quoted run.
+ */
+ private static boolean isSafeUnloadSubquery(String q) {
+ if (!(startsWithKeyword(q, "SELECT") || startsWithKeyword(q, "WITH"))) {
+ return false;
+ }
+ int depth = 0;
+ boolean inSingle = false;
+ boolean inDouble = false;
+ for (int i = 0; i < q.length(); i++) {
+ char c = q.charAt(i);
+ if (inSingle) {
+ if (c == '\'') {
+ if (i + 1 < q.length() && q.charAt(i + 1) == '\'') {
+ i++;
+ } else {
+ inSingle = false;
+ }
+ }
+ continue;
+ }
+ if (inDouble) {
+ if (c == '"') {
+ inDouble = false;
+ }
+ continue;
+ }
+ switch (c) {
+ case '\'' -> {
+ if (i > 0) {
+ char prev = q.charAt(i - 1);
+ if (prev == 'E' || prev == 'e') {
+ return false; // C-style escape string literal
+ }
+ if (prev == '&' && i > 1) {
+ char prev2 = q.charAt(i - 2);
+ if (prev2 == 'U' || prev2 == 'u') {
+ return false; // Unicode escape string literal
+ }
+ }
+ }
+ inSingle = true;
+ }
+ case '"' -> {
+ if (i > 1 && q.charAt(i - 1) == '&') {
+ char prev2 = q.charAt(i - 2);
+ if (prev2 == 'U' || prev2 == 'u') {
+ return false; // Unicode escape identifier
+ }
+ }
+ inDouble = true;
+ }
+ case '\\' -> {
+ return false; // backslash outside quotes
+ }
+ case '(' -> depth++;
+ case ')' -> {
+ if (--depth < 0) {
+ return false;
+ }
+ }
+ case ';', '$' -> {
+ return false;
+ }
+ case '-' -> {
+ if (i + 1 < q.length() && q.charAt(i + 1) == '-') {
+ return false;
+ }
+ }
+ case '/' -> {
+ if (i + 1 < q.length() && q.charAt(i + 1) == '*') {
+ return false;
+ }
+ }
+ default -> {
+ }
+ }
+ }
+ return depth == 0 && !inSingle && !inDouble;
+ }
+
+ private static boolean startsWithKeyword(String s, String keyword) {
+ String t = s.stripLeading();
+ if (!t.regionMatches(true, 0, keyword, 0, keyword.length())) {
+ return false;
+ }
+ if (t.length() == keyword.length()) {
+ return true;
+ }
+ char next = t.charAt(keyword.length());
+ return !(Character.isLetterOrDigit(next) || next == '_' || next == '$');
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
index 85d495e4db..db9c9bcf5c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
@@ -102,18 +102,23 @@ public class RedshiftInterceptingBridge {
String sql = msg.getSql();
- CopyStatementParser.S3CopyFrom copyFrom = null;
+ CopyStatementParser.S3Statement parsed = null;
try {
- copyFrom = CopyStatementParser.parse(sql);
+ parsed = CopyStatementParser.parse(sql);
} catch (RuntimeException e) {
LOG.warnv("CopyStatementParser failed, forwarding original query: {0}", e.getMessage());
}
- if (copyFrom != null) {
- CopyStatementParser.S3CopyFrom spec = copyFrom;
- boolean intercepted = runWithBackendOwned(
- () -> S3CopySimulator.runCopyFrom(client, backend, spec, s3Service, lastBackendStatus,
- status -> lastBackendStatus = (char) status));
+ if (parsed != null) {
+ CopyStatementParser.S3Statement statement = parsed;
+ boolean intercepted = runWithBackendOwned(() -> switch (statement) {
+ case CopyStatementParser.S3CopyFrom c ->
+ S3CopySimulator.runCopyFrom(client, backend, c, s3Service, lastBackendStatus,
+ status -> lastBackendStatus = (char) status);
+ case CopyStatementParser.S3Unload u ->
+ S3CopySimulator.runUnload(client, backend, u, s3Service, lastBackendStatus,
+ status -> lastBackendStatus = (char) status);
+ });
if (intercepted) {
continue;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
index 3ff45c0f6d..a2145c3ccd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
@@ -14,12 +14,17 @@ import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Semaphore;
+import java.util.function.IntConsumer;
import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
/**
- * Emulates {@code COPY <table> FROM 's3://bucket/keyOrPrefix'} by reading the S3 object (or every
- * object under the prefix, in key order) through {@link S3Service} and streaming it into the
- * backing PostgreSQL container over a fabricated {@code COPY <table> FROM STDIN} exchange.
+ * Emulates {@code COPY <table> FROM 's3://bucket/keyOrPrefix'} and
+ * {@code UNLOAD ('<select>') TO 's3://bucket/prefix'} by streaming data between {@link S3Service}
+ * and the backing PostgreSQL container over a fabricated {@code COPY ... FROM STDIN} or
+ * {@code COPY (...) TO STDOUT} exchange.
*
* <p>The caller must own the backend socket exclusively for the whole call: this class writes a
* query and reads its response frames directly. Every exit path leaves the client with exactly one
@@ -33,6 +38,20 @@ public final class S3CopySimulator {
private static final int LIST_PAGE_SIZE = 1000;
private static final int CHUNK = 8192;
+ // The two limits below are package-private and non-final only so unit tests can shrink them to
+ // force multi-slice and over-limit paths on small inputs. Surefire runs a test class serially, and
+ // each test restores the previous value in a finally block, so there is no cross-test interference.
+ /** Default per-object size when the statement gives no MAXFILESIZE. Small because each slice is buffered in heap. */
+ static long UNLOAD_TARGET_FILE_BYTES = 6L * 1024 * 1024;
+ /** Whole-result ceiling; a larger UNLOAD is aborted rather than filling the in-memory S3 store. */
+ static long UNLOAD_MAX_TOTAL_BYTES = 256L * 1024 * 1024;
+ private static final long UNLOAD_WARN_BYTES = 32L * 1024 * 1024;
+ /** Shared across every connection so concurrent UNLOADs cannot multiply the per-slice heap cost without bound. */
+ static final Semaphore UNLOAD_HEAP_MIB = new Semaphore(192);
+ private static final int UNLOAD_INITIAL_MIB = 12;
+
+ private static final String SQLSTATE_PROGRAM_LIMIT_EXCEEDED = "54000";
+ private static final String SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED = "53400";
private static final String SQLSTATE_INSUFFICIENT_PRIVILEGE = "42501";
private static final String SQLSTATE_INTERNAL = "XX000";
@@ -54,7 +73,7 @@ public final class S3CopySimulator {
public static boolean runCopyFrom(Socket client, Socket backend,
CopyStatementParser.S3CopyFrom spec, S3Service s3,
- char txStatus, java.util.function.IntConsumer onStatusChange) throws IOException {
+ char txStatus, IntConsumer onStatusChange) throws IOException {
try {
s3.authorizeAnonymousListBucket(spec.bucket());
} catch (AwsException e) {
@@ -135,7 +154,7 @@ public final class S3CopySimulator {
private static void abortOpenCopyIn(Socket client, Socket backend, OutputStream backendOut,
PostgresWireDecoder backendDecoder, Exception cause,
- char txStatus, java.util.function.IntConsumer onStatusChange) throws IOException {
+ char txStatus, IntConsumer onStatusChange) throws IOException {
try {
writeCopyFail(backendOut, cause.getMessage());
drainToReadyForQuery(backendDecoder, client, onStatusChange);
@@ -276,7 +295,7 @@ public final class S3CopySimulator {
}
private static void drainToReadyForQuery(PostgresWireDecoder decoder, Socket client,
- java.util.function.IntConsumer onStatusChange) throws IOException {
+ IntConsumer onStatusChange) throws IOException {
OutputStream clientOut = client.getOutputStream();
while (true) {
PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();
@@ -300,13 +319,13 @@ public final class S3CopySimulator {
}
private static void sendAccessDenied(Socket client, Socket backend, CopyStatementParser.S3CopyFrom spec,
- char txStatus, java.util.function.IntConsumer onStatusChange) throws IOException {
+ char txStatus, IntConsumer onStatusChange) throws IOException {
sendError(client, backend, SQLSTATE_INSUFFICIENT_PRIVILEGE,
"S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), txStatus, onStatusChange);
}
private static void sendError(Socket client, Socket backend, String sqlState, String message,
- char txStatus, java.util.function.IntConsumer onStatusChange) throws IOException {
+ char txStatus, IntConsumer onStatusChange) throws IOException {
if (txStatus == 'T' && backend != null && !backend.isClosed()) {
if (!failBackendTransaction(backend, onStatusChange)) {
closeQuietly(backend);
@@ -340,7 +359,7 @@ public final class S3CopySimulator {
}
}
- private static boolean failBackendTransaction(Socket backend, java.util.function.IntConsumer onStatusChange) {
+ private static boolean failBackendTransaction(Socket backend, IntConsumer onStatusChange) {
try {
OutputStream out = backend.getOutputStream();
out.write(PostgresWireDecoder.encodeQuery("(FLOCI_ABORT_TX)"));
@@ -401,4 +420,459 @@ public final class S3CopySimulator {
(byte) (value >>> 24), (byte) (value >>> 16), (byte) (value >>> 8), (byte) value
};
}
+
+ public static boolean runUnload(Socket client, Socket backend,
+ CopyStatementParser.S3Unload spec, S3Service s3, char txStatus) throws IOException {
+ return runUnload(client, backend, spec, s3, txStatus, null);
+ }
+
+ public static boolean runUnload(Socket client, Socket backend,
+ CopyStatementParser.S3Unload spec, S3Service s3, char txStatus,
+ IntConsumer onStatusChange) throws IOException {
+
+ String probeKey = unloadDataKey(spec, 0);
+ String manifestKey = spec.prefix() + "manifest";
+ try {
+ s3.authorizeAnonymousPutObject(spec.bucket(), probeKey);
+ if (spec.manifest()) {
+ s3.authorizeAnonymousPutObject(spec.bucket(), manifestKey);
+ }
+ } catch (AwsException e) {
+ LOG.debugv(e, "PutObject denied for UNLOAD to s3://{0}/{1}", spec.bucket(), spec.prefix());
+ sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);
+ return true;
+ }
+
+ try {
+ if (!spec.allowOverwrite()) {
+ s3.authorizeAnonymousListBucket(spec.bucket());
+ if (targetPrefixHasObjects(spec, s3)) {
+ sendError(client, backend, SQLSTATE_INTERNAL,
+ "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()
+ + " is not empty; specify ALLOWOVERWRITE to overwrite", txStatus, onStatusChange);
+ return true;
+ }
+ }
+ } catch (AwsException e) {
+ LOG.debugv(e, "bucket check failed for UNLOAD to s3://{0}/{1}", spec.bucket(), spec.prefix());
+ sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);
+ return true;
+ }
+
+ if (!UNLOAD_HEAP_MIB.tryAcquire(UNLOAD_INITIAL_MIB)) {
+ sendError(client, backend, SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,
+ "UNLOAD memory budget exhausted; retry shortly", txStatus, onStatusChange);
+ return true;
+ }
+ int[] heldMib = {UNLOAD_INITIAL_MIB};
+ try {
+ return runUnloadStreaming(client, backend, spec, s3, txStatus, onStatusChange, heldMib);
+ } finally {
+ UNLOAD_HEAP_MIB.release(heldMib[0]);
+ }
+ }
+
+ private static boolean runUnloadStreaming(Socket client, Socket backend,
+ CopyStatementParser.S3Unload spec, S3Service s3, char txStatus,
+ IntConsumer onStatusChange, int[] heldMib) throws IOException {
+
+ PostgresWireDecoder.FrontendMessage cmdComplete = null;
+ PostgresWireDecoder.FrontendMessage readyForQuery = null;
+
+ OutputStream backendOut = backend.getOutputStream();
+ backendOut.write(PostgresWireDecoder.encodeQuery(fabricateUnloadCopy(spec)));
+ backendOut.flush();
+
+ PostgresWireDecoder backendDecoder = new PostgresWireDecoder(backend.getInputStream());
+ PostgresWireDecoder.FrontendMessage first;
+ try {
+ first = nextNonAsync(backendDecoder, client);
+ } catch (IOException e) {
+ LOG.warnv(e, "backend read failed while awaiting CopyOutResponse");
+ closeQuietly(backend);
+ sendError(client, null, SQLSTATE_INTERNAL,
+ "UNLOAD failed: backend closed or timed out", txStatus, onStatusChange);
+ closeQuietly(client);
+ return true;
+ }
+ if (first == null) {
+ closeQuietly(backend);
+ sendError(client, null, SQLSTATE_INTERNAL,
+ "UNLOAD failed: backend closed before COPY started", txStatus, onStatusChange);
+ closeQuietly(client);
+ return true;
+ }
+ if (first.type() != 'H') {
+ // Backend rejected the SELECT itself. Its ErrorResponse + ReadyForQuery are the client's one response.
+ forward(client, first);
+ drainToReadyForQuery(backendDecoder, client, onStatusChange);
+ return true;
+ }
+
+ long threshold = spec.maxFileSizeBytes() > 0 ? spec.maxFileSizeBytes() : UNLOAD_TARGET_FILE_BYTES;
+ String contentType = spec.gzip() ? "application/gzip" : "text/plain";
+ List<String> writtenKeys = new ArrayList<>();
+ List<Integer> writtenLengths = new ArrayList<>();
+
+ ByteArrayOutputStream sink = new ByteArrayOutputStream();
+ OutputStream acc = spec.gzip() ? new GZIPOutputStream(sink) : sink;
+ long rawSlice = 0;
+ // Data-row bytes in the current slice, excluding a repeated header. Drives the size split so a
+ // header-only slice is never cut off on its own.
+ long slicePayload = 0;
+ long rawTotal = 0;
+ boolean lastByteNewline = true;
+ boolean warned = false;
+ int sliceIndex = 0;
+
+ // With HEADER the backend emits the header row once, at the top of the stream. Capture it so
+ // it can be repeated at the start of every later slice, the way real Redshift writes one
+ // header per output file.
+ ByteArrayOutputStream headerBuf = spec.header() ? new ByteArrayOutputStream() : null;
+ byte[] headerBytes = null;
+ boolean capturingHeader = spec.header();
+
+ try {
+ while (true) {
+ PostgresWireDecoder.FrontendMessage m = backendDecoder.nextMessage();
+ if (m == null) {
+ closeAcc(acc, sink);
+ closeQuietly(backend);
+ if (spec.manifest()) {
+ deleteWritten(s3, spec, writtenKeys);
+ }
+ sendError(client, null, SQLSTATE_INTERNAL,
+ "UNLOAD failed: backend closed mid-stream", txStatus, onStatusChange);
+ closeQuietly(client);
+ return true;
+ }
+ char t = m.type();
+ if (t == 'd') {
+ byte[] body = m.body();
+ int dataStart = 0;
+ if (capturingHeader) {
+ int nl = -1;
+ for (int i = 0; i < body.length; i++) {
+ if (body[i] == '\n') {
+ nl = i;
+ break;
+ }
+ }
+ int end = nl >= 0 ? nl + 1 : body.length;
+ headerBuf.write(body, 0, end);
+ if (nl >= 0) {
+ headerBytes = headerBuf.toByteArray();
+ headerBuf = null;
+ capturingHeader = false;
+ }
+ dataStart = end;
+ } else if (headerBytes != null && sliceIndex > 0 && rawSlice == 0) {
+ // First data of a later slice: repeat the captured header row.
+ acc.write(headerBytes);
+ rawSlice += headerBytes.length;
+ }
+ acc.write(body, 0, body.length);
+ rawSlice += body.length;
+ rawTotal += body.length;
+ slicePayload += body.length - dataStart;
+ if (body.length > 0) {
+ lastByteNewline = body[body.length - 1] == '\n';
+ }
+ if (!warned && rawTotal > UNLOAD_WARN_BYTES) {
+ warned = true;
+ LOG.warnv("UNLOAD result passed {0} bytes and is buffered a slice at a time "
+ + "(bucket={1}, prefix={2})", UNLOAD_WARN_BYTES, spec.bucket(), spec.prefix());
+ }
+ int wantMib = (int) ((rawSlice * 3) / (1024 * 1024)) + 1;
+ while (heldMib[0] < wantMib) {
+ if (!UNLOAD_HEAP_MIB.tryAcquire(1)) {
+ return abortUnload(client, backend, backendDecoder, acc, sink,
+ SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,
+ "UNLOAD memory budget exhausted; retry with a smaller result",
+ txStatus, onStatusChange);
+ }
+ heldMib[0]++;
+ }
+ if (rawTotal > UNLOAD_MAX_TOTAL_BYTES) {
+ closeAcc(acc, sink);
+ closeQuietly(backend);
+ if (spec.manifest()) {
+ deleteWritten(s3, spec, writtenKeys);
+ }
+ sendError(client, null,
+ SQLSTATE_PROGRAM_LIMIT_EXCEEDED,
+ "UNLOAD result exceeds the " + UNLOAD_MAX_TOTAL_BYTES + "-byte limit",
+ txStatus, onStatusChange);
+ closeQuietly(client);
+ return true;
+ }
+ if (slicePayload >= threshold && lastByteNewline) {
+ byte[] payload = finishSlice(acc, sink);
+ String key = unloadDataKey(spec, sliceIndex);
+ try {
+ s3.authorizeAnonymousPutObject(spec.bucket(), key);
+ s3.putObject(spec.bucket(), key, payload, contentType, Map.of());
+ } catch (AwsException e) {
+ LOG.debugv(e, "UNLOAD slice write to s3://{0}/{1} failed", spec.bucket(), key);
+ if (spec.manifest()) {
+ deleteWritten(s3, spec, writtenKeys);
+ }
+ return abortUnload(client, backend, backendDecoder, sink, sink,
+ unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);
+ }
+ writtenKeys.add(key);
+ writtenLengths.add(payload.length);
+ sliceIndex++;
+ int release = heldMib[0] - UNLOAD_INITIAL_MIB;
+ if (release > 0) {
+ UNLOAD_HEAP_MIB.release(release);
+ heldMib[0] = UNLOAD_INITIAL_MIB;
+ }
+ rawSlice = 0;
+ slicePayload = 0;
+ lastByteNewline = true;
+ sink = new ByteArrayOutputStream();
+ acc = spec.gzip() ? new GZIPOutputStream(sink) : sink;
+ }
+ } else if (t == 'c') {
+ // CopyDone, ignore
+ } else if (t == 'C') {
+ // CommandComplete, hold until we see 'Z'
+ cmdComplete = m;
+ } else if (t == 'Z') {
+ readyForQuery = m;
+ break;
+ } else if (t == 'E') {
+ closeAcc(acc, sink);
+ if (spec.manifest()) {
+ deleteWritten(s3, spec, writtenKeys);
+ }
+ forward(client, m);
+ drainToReadyForQuery(backendDecoder, client, onStatusChange);
+ return true;
+ } else if (t == 'N' || t == 'A' || t == 'S') {
+ forward(client, m);
+ }
+ }
+ } catch (AwsException e) {
+ LOG.debugv(e, "UNLOAD S3 operation failed during streaming");
+ if (spec.manifest()) {
+ deleteWritten(s3, spec, writtenKeys);
+ }
+ return abortUnload(client, backend, backendDecoder, acc, sink,
+ unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);
+ } catch (RuntimeException | IOException e) {
+ LOG.warnv(e, "UNLOAD streaming failed");
+ if (spec.manifest()) {
+ deleteWritten(s3, spec, writtenKeys);
+ }
+ String detail = e.getMessage() != null ? e.getMessage() : e.toString();
+ return abortUnload(client, backend, backendDecoder, acc, sink,
+ SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, txStatus, onStatusChange);
+ }
+
+ // Flush the final slice. Write it when it carries rows, or when nothing has been written yet
+ // (a zero-row UNLOAD still emits one object, header only when HEADER was asked for). A result
+ // that ended exactly on a slice boundary leaves slicePayload == 0 with slices already written,
+ // so neither an empty nor a header-only trailer is produced.
+ byte[] payload = finishSlice(acc, sink);
+ if (slicePayload > 0 || writtenKeys.isEmpty()) {
+ String key = unloadDataKey(spec, sliceIndex);
+ try {
+ s3.authorizeAnonymousPutObject(spec.bucket(), key);
+ s3.putObject(spec.bucket(), key, payload, contentType, Map.of());
+ } catch (AwsException e) {
+ LOG.debugv(e, "UNLOAD final slice write to s3://{0}/{1} failed", spec.bucket(), key);
+ if (spec.manifest()) {
+ deleteWritten(s3, spec, writtenKeys);
+ }
+ sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec),
+ txStatus, onStatusChange);
+ return true;
+ } catch (RuntimeException e) {
+ LOG.warnv(e, "UNLOAD final slice write failed");
+ if (spec.manifest()) {
+ deleteWritten(s3, spec, writtenKeys);
+ }
+ String detail = e.getMessage() != null ? e.getMessage() : e.toString();
+ sendError(client, backend, SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, txStatus, onStatusChange);
+ return true;
+ }
+ writtenKeys.add(key);
+ writtenLengths.add(payload.length);
+ }
+
+ if (spec.manifest()) {
+ try {
+ s3.authorizeAnonymousPutObject(spec.bucket(), spec.prefix() + "manifest");
+ s3.putObject(spec.bucket(), spec.prefix() + "manifest",
+ manifestJson(spec.bucket(), writtenKeys, writtenLengths).getBytes(StandardCharsets.UTF_8),
+ "application/json", Map.of());
+ } catch (RuntimeException e) {
+ LOG.warnv(e, "UNLOAD manifest write failed; removing partial data objects");
+ deleteWritten(s3, spec, writtenKeys);
+ sendError(client, backend, SQLSTATE_INTERNAL, "UNLOAD manifest write failed", txStatus, onStatusChange);
+ return true;
+ }
+ }
+
+ if (cmdComplete != null) {
+ forward(client, cmdComplete);
+ }
+ forward(client, readyForQuery);
+ if (onStatusChange != null && readyForQuery.body().length > 0) {
+ onStatusChange.accept(readyForQuery.body()[0]);
+ }
+ return true;
+ }
+
+ private static boolean targetPrefixHasObjects(CopyStatementParser.S3Unload spec, S3Service s3) {
+ S3Service.ListObjectsResult r = s3.listObjectsWithPrefixes(
+ spec.bucket(), spec.prefix(), null, 1, null, null);
+ return r != null && r.objects() != null && !r.objects().isEmpty();
+ }
+
+ private static String fabricateUnloadCopy(CopyStatementParser.S3Unload spec) {
+ boolean csvFraming = spec.csv() || spec.addQuotes() || spec.header();
+ StringBuilder sql = new StringBuilder("COPY (").append(spec.selectQuery())
+ .append(") TO STDOUT WITH (FORMAT ").append(csvFraming ? "csv" : "text");
+ String delimiter = spec.delimiter() != null ? spec.delimiter() : (csvFraming ? "," : "|");
+ sql.append(", DELIMITER '").append(quoteLiteral(delimiter)).append("'");
+ if (spec.header()) {
+ sql.append(", HEADER true");
+ }
+ if (spec.addQuotes()) {
+ sql.append(", FORCE_QUOTE *");
+ }
+ if (spec.nullAs() != null) {
+ sql.append(", NULL '").append(quoteLiteral(spec.nullAs())).append("'");
+ }
+ sql.append(")");
+ return sql.toString();
+ }
+
+ /** {@code 404} from the S3 layer means the bucket is absent, not that access was denied. */
+ private static String unloadWriteSqlState(AwsException e) {
+ return e.getHttpStatus() == 404 ? SQLSTATE_INTERNAL : SQLSTATE_INSUFFICIENT_PRIVILEGE;
+ }
+
+ private static String unloadWriteMessage(AwsException e, CopyStatementParser.S3Unload spec) {
+ return e.getHttpStatus() == 404
+ ? "S3 bucket s3://" + spec.bucket() + " not found"
+ : "S3 access denied for s3://" + spec.bucket() + "/" + spec.prefix();
+ }
+
+ private static String unloadDataKey(CopyStatementParser.S3Unload spec, int index) {
+ String base = spec.parallel()
+ ? spec.prefix() + String.format("%04d_part_00", index)
+ : spec.prefix() + String.format("%03d", index);
+ return spec.gzip() ? base + ".gz" : base;
+ }
+
+ private static byte[] finishSlice(OutputStream acc, ByteArrayOutputStream sink) throws IOException {
+ if (acc != sink) {
+ acc.close(); // flush the GZIP trailer
+ }
+ return sink.toByteArray();
+ }
+
+ private static void closeAcc(OutputStream acc, ByteArrayOutputStream sink) {
+ if (acc != sink) {
+ try {
+ acc.close();
+ } catch (IOException e) {
+ LOG.debugv(e, "error closing the UNLOAD accumulation stream");
+ }
+ }
+ }
+
+ private static boolean abortUnload(Socket client, Socket backend, PostgresWireDecoder backendDecoder,
+ OutputStream acc, ByteArrayOutputStream sink, String sqlState, String message,
+ char txStatus, IntConsumer onStatusChange) throws IOException {
+ closeAcc(acc, sink);
+ try {
+ drainBackendDiscarding(backendDecoder);
+ } catch (IOException backendGone) {
+ // The backend is unreachable, so it cannot be resynchronized: close it and hand the
+ // client its one response synthesized, exactly as the backend-EOF branch does.
+ LOG.debugv(backendGone, "backend unreachable while draining an aborted UNLOAD; synthesizing client error");
+ closeQuietly(backend);
+ sendError(client, null, sqlState, message, txStatus, onStatusChange);
+ closeQuietly(client);
+ return true;
+ }
+ sendError(client, backend, sqlState, message, txStatus, onStatusChange);
+ return true;
+ }
+
+ /** Read backend messages, forwarding nothing, until its ReadyForQuery or EOF. Leaves the backend socket clean. */
+ private static void drainBackendDiscarding(PostgresWireDecoder decoder) throws IOException {
+ PostgresWireDecoder.FrontendMessage m;
+ while ((m = decoder.nextMessage()) != null) {
+ if (m.type() == 'Z') {
+ return;
+ }
+ }
+ }
+
+ private static void deleteWritten(S3Service s3, CopyStatementParser.S3Unload spec, List<String> keys) {
+ if (spec.allowOverwrite()) {
+ // Under ALLOWOVERWRITE a slice key may have replaced a pre-existing object, and there is no
+ // reliable way to tell an overwrite from a fresh write (the check and the write are not
+ // atomic, and a concurrent UNLOAD to the same prefix could own the stored object). Rather
+ // than risk deleting data this operation did not create, leave every written object in
+ // place. A failed MANIFEST + ALLOWOVERWRITE UNLOAD can therefore leave valid data objects
+ // without a manifest; rerunning the same statement overwrites them cleanly.
+ return;
+ }
+ // Non-overwrite runs verified the prefix was empty before streaming, so every written key was
+ // created by this operation and is safe to remove.
+ for (String k : keys) {
+ try {
+ s3.authorizeAnonymousDeleteObject(spec.bucket(), k);
+ s3.deleteObject(spec.bucket(), k);
+ } catch (RuntimeException e) {
+ LOG.debugv(e, "could not remove partial UNLOAD object s3://{0}/{1}", spec.bucket(), k);
+ }
+ }
+ }
+
+ private static String manifestJson(String bucket, List<String> keys, List<Integer> lengths) {
+ StringBuilder json = new StringBuilder("{\"entries\":[");
+ for (int i = 0; i < keys.size(); i++) {
+ if (i > 0) {
+ json.append(',');
+ }
+ json.append("{\"url\":\"s3://").append(escapeJson(bucket)).append('/').append(escapeJson(keys.get(i)))
+ .append("\",\"meta\":{\"content_length\":").append(lengths.get(i)).append("}}");
+ }
+ return json.append("]}").toString();
+ }
+
+ private static String escapeJson(String s) {
+ if (s == null) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder(s.length() + 8);
+ for (int i = 0; i < s.length(); i++) {
+ char c = s.charAt(i);
+ switch (c) {
+ case '"' -> sb.append("\\\"");
+ case '\\' -> sb.append("\\\\");
+ case '\b' -> sb.append("\\b");
+ case '\f' -> sb.append("\\f");
+ case '\n' -> sb.append("\\n");
+ case '\r' -> sb.append("\\r");
+ case '\t' -> sb.append("\\t");
+ default -> {
+ if (c < ' ') {
+ sb.append(String.format("\\u%04x", (int) c));
+ } else {
+ sb.append(c);
+ }
+ }
+ }
+ }
+ return sb.toString();
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index de6a63d597..490e919417 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -617,6 +617,16 @@ public class S3Service implements Resettable, ResourceProvider {
authorizeListBucket(bucketName, RequestAuthorization.unsigned());
}
+ /** Authorize an unsigned {@code s3:PutObject}; see {@link #authorizeAnonymousGetObject}. */
+ public void authorizeAnonymousPutObject(String bucketName, String key) {
+ authorizePutObject(bucketName, key, RequestAuthorization.unsigned());
+ }
+
+ /** Authorize an unsigned {@code s3:DeleteObject}; see {@link #authorizeAnonymousGetObject}. */
+ public void authorizeAnonymousDeleteObject(String bucketName, String key) {
+ authorizeDeleteObject(bucketName, key, null, RequestAuthorization.unsigned());
+ }
+
public void authorizeCloudFrontOacGetObject(
String bucketName, String key, String distributionArn) {
authorizeCloudFrontGetObject(
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java
index 9b12d1ff44..9fe447dd54 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java
@@ -1,6 +1,8 @@
package io.github.hectorvent.floci.services.redshift;
import io.github.hectorvent.floci.services.redshift.model.Cluster;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
import io.quarkus.test.junit.QuarkusTest;
import jakarta.inject.Inject;
import org.awaitility.Awaitility;
@@ -10,15 +12,25 @@ import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
import java.util.Objects;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@QuarkusTest
@@ -28,7 +40,7 @@ class RedshiftInterceptorIntegrationTest {
RedshiftService service;
@Inject
- io.github.hectorvent.floci.services.s3.S3Service s3;
+ S3Service s3;
private String clusterId;
@@ -118,7 +130,7 @@ class RedshiftInterceptorIntegrationTest {
String bucket = "redshift-copy-it";
s3.createBucket(bucket, "us-east-1");
s3.putObject(bucket, "people/p1.txt",
- "1|alice\n2|bob\n".getBytes(java.nio.charset.StandardCharsets.UTF_8), "text/plain", java.util.Map.of());
+ "1|alice\n2|bob\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {
c.createStatement().execute("CREATE TABLE people (id int, name text)");
@@ -137,8 +149,8 @@ class RedshiftInterceptorIntegrationTest {
String bucket = "redshift-copy-it-prefix";
s3.createBucket(bucket, "us-east-1");
- s3.putObject(bucket, "d/a", "1|a\n".getBytes(java.nio.charset.StandardCharsets.UTF_8), "text/plain", java.util.Map.of());
- s3.putObject(bucket, "d/b", "2|b\n".getBytes(java.nio.charset.StandardCharsets.UTF_8), "text/plain", java.util.Map.of());
+ s3.putObject(bucket, "d/a", "1|a\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+ s3.putObject(bucket, "d/b", "2|b\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {
c.createStatement().execute("CREATE TABLE t (id int, v text)");
@@ -157,11 +169,11 @@ class RedshiftInterceptorIntegrationTest {
String bucket = "redshift-copy-it-gzip";
s3.createBucket(bucket, "us-east-1");
- java.io.ByteArrayOutputStream raw = new java.io.ByteArrayOutputStream();
- try (java.util.zip.GZIPOutputStream gz = new java.util.zip.GZIPOutputStream(raw)) {
- gz.write("10|x\n11|y\n".getBytes(java.nio.charset.StandardCharsets.UTF_8));
+ ByteArrayOutputStream raw = new ByteArrayOutputStream();
+ try (GZIPOutputStream gz = new GZIPOutputStream(raw)) {
+ gz.write("10|x\n11|y\n".getBytes(StandardCharsets.UTF_8));
}
- s3.putObject(bucket, "g/data.gz", raw.toByteArray(), "application/gzip", java.util.Map.of());
+ s3.putObject(bucket, "g/data.gz", raw.toByteArray(), "application/gzip", Map.of());
try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {
c.createStatement().execute("CREATE TABLE g (id int, v text)");
@@ -183,7 +195,7 @@ class RedshiftInterceptorIntegrationTest {
try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {
c.createStatement().execute("CREATE TABLE t2 (id int)");
- SQLException ex = org.junit.jupiter.api.Assertions.assertThrows(
+ SQLException ex = assertThrows(
SQLException.class,
() -> c.createStatement().execute("COPY t2 FROM 's3://redshift-copy-it-missing/does/not/exist'"));
assertTrue(
@@ -204,12 +216,12 @@ class RedshiftInterceptorIntegrationTest {
c.setAutoCommit(false);
Statement st = c.createStatement();
st.execute("INSERT INTO tx_test VALUES (1)");
- org.junit.jupiter.api.Assertions.assertThrows(
+ assertThrows(
SQLException.class,
() -> st.execute("COPY tx_test FROM 's3://redshift-copy-it-tx/does/not/exist'"));
// The backend must now be in the aborted transaction state ('E').
// Subsequent statements in this transaction block must fail.
- org.junit.jupiter.api.Assertions.assertThrows(
+ assertThrows(
SQLException.class,
() -> st.execute("INSERT INTO tx_test VALUES (2)"));
c.rollback();
@@ -220,4 +232,99 @@ class RedshiftInterceptorIntegrationTest {
}
}
}
+
+ @Test
+ void unloadToS3PrefixWritesRowsAsObjects() throws Exception {
+ clusterId = "it-unload-basic";
+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+
+ String bucket = "redshift-unload-it";
+ s3.createBucket(bucket, "us-east-1");
+
+ try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {
+ c.createStatement().execute("CREATE TABLE u_src (id int, name text)");
+ c.createStatement().execute("INSERT INTO u_src VALUES (1, 'alice'), (2, 'bob')");
+ c.createStatement().execute(
+ "UNLOAD ('select id, name from u_src order by id') TO 's3://redshift-unload-it/out/'");
+ }
+
+ List<S3Object> objs =
+ s3.listObjects(bucket, "out/", null, 100);
+ assertTrue(objs.size() >= 1, "UNLOAD must write at least one object");
+ StringBuilder all = new StringBuilder();
+ objs.stream()
+ .sorted(Comparator.comparing(S3Object::getKey))
+ // listObjects returns metadata-only entries; fetch each body with getObject.
+ .forEach(o -> all.append(new String(
+ s3.getObject(bucket, o.getKey()).getData(), StandardCharsets.UTF_8)));
+ assertEquals("1|alice\n2|bob\n", all.toString());
+ }
+
+ @Test
+ void unloadGzipWritesCompressedObject() throws Exception {
+ clusterId = "it-unload-gzip";
+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+ String bucket = "redshift-unload-it-gzip";
+ s3.createBucket(bucket, "us-east-1");
+
+ try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {
+ c.createStatement().execute("CREATE TABLE g_src (id int)");
+ c.createStatement().execute("INSERT INTO g_src VALUES (7), (8)");
+ c.createStatement().execute("UNLOAD ('select id from g_src order by id') TO 's3://redshift-unload-it-gzip/g/' GZIP");
+ }
+
+ var objs = s3.listObjects(bucket, "g/", null, 100);
+ assertEquals(1, objs.size());
+ assertTrue(objs.get(0).getKey().endsWith(".gz"), objs.get(0).getKey());
+ byte[] data = s3.getObject(bucket, objs.get(0).getKey()).getData();
+ ByteArrayOutputStream raw = new ByteArrayOutputStream();
+ try (var in = new GZIPInputStream(new ByteArrayInputStream(data))) {
+ in.transferTo(raw);
+ }
+ assertEquals("7\n8\n", raw.toString(StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void unloadWithManifestWritesAManifestObject() throws Exception {
+ clusterId = "it-unload-manifest";
+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+ String bucket = "redshift-unload-it-manifest";
+ s3.createBucket(bucket, "us-east-1");
+
+ try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {
+ c.createStatement().execute("CREATE TABLE m_src (id int)");
+ c.createStatement().execute("INSERT INTO m_src VALUES (1)");
+ c.createStatement().execute("UNLOAD ('select id from m_src') TO 's3://redshift-unload-it-manifest/m/' MANIFEST");
+ }
+
+ var manifest = s3.getObject(bucket, "m/manifest");
+ assertNotNull(manifest);
+ String json = new String(manifest.getData(), StandardCharsets.UTF_8);
+ assertTrue(json.contains("\"entries\""), json);
+ assertTrue(json.contains("s3://redshift-unload-it-manifest/m/"), json);
+ }
+
+ @Test
+ void unloadIntoNonEmptyPrefixWithoutAllowOverwriteRaisesSqlError() throws Exception {
+ clusterId = "it-unload-overwrite";
+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+ String bucket = "redshift-unload-it-ow";
+ s3.createBucket(bucket, "us-east-1");
+ s3.putObject(bucket, "o/existing", "x".getBytes(StandardCharsets.UTF_8),
+ "text/plain", Map.of());
+
+ try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {
+ c.createStatement().execute("CREATE TABLE o_src (id int)");
+ c.createStatement().execute("INSERT INTO o_src VALUES (1)");
+ SQLException ex = assertThrows(SQLException.class,
+ () -> c.createStatement().execute("UNLOAD ('select id from o_src') TO 's3://redshift-unload-it-ow/o/'"));
+ assertTrue(ex.getMessage().toLowerCase().contains("allowoverwrite"), ex.getMessage());
+ }
+
+ // With ALLOWOVERWRITE the same statement succeeds.
+ try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {
+ c.createStatement().execute(
+ "UNLOAD ('select id from o_src') TO 's3://redshift-unload-it-ow/o/' ALLOWOVERWRITE");
+ }
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java
index c47a26f99b..a7ecb616ac 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java
@@ -17,6 +17,7 @@ import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.containsString;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.when;
@QuarkusTest
@@ -110,7 +111,7 @@ public class RedshiftOperationsTest {
void testClusterAndSnapshotLifecycle() {
when(containerManager.start(any(), eq("cluster-src"), any(), any()))
.thenReturn(new RedshiftContainerHandle("c1", "cluster-src", "localhost", 5439));
- org.mockito.Mockito.doAnswer(invocation -> {
+ doAnswer(invocation -> {
Path p = invocation.getArgument(3);
Files.writeString(p, "-- dump sql table test_data;");
return null;
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java
index 06807ed2b0..1fb0e2bd0f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.redshift;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.redshift.model.Cluster;
import io.github.hectorvent.floci.services.redshift.model.ClusterParameterGroup;
import io.github.hectorvent.floci.services.redshift.model.ClusterSubnetGroup;
@@ -17,6 +18,7 @@ import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.*;
@@ -334,8 +336,8 @@ class RedshiftQueryHandlerTest {
MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
params.putSingle("Description", "desc");
- io.github.hectorvent.floci.core.common.AwsException ex = org.junit.jupiter.api.Assertions.assertThrows(
- io.github.hectorvent.floci.core.common.AwsException.class,
+ AwsException ex = assertThrows(
+ AwsException.class,
() -> handler.handle("CreateClusterSubnetGroup", params));
assertEquals("InvalidParameterValue", ex.getErrorCode());
}
@@ -345,8 +347,8 @@ class RedshiftQueryHandlerTest {
MultivaluedMap<String, String> params = new MultivaluedHashMap<>();
params.putSingle("NodeType", "dc2.large");
- io.github.hectorvent.floci.core.common.AwsException ex = org.junit.jupiter.api.Assertions.assertThrows(
- io.github.hectorvent.floci.core.common.AwsException.class,
+ AwsException ex = assertThrows(
+ AwsException.class,
() -> handler.handle("ModifyCluster", params));
assertEquals("InvalidParameterValue", ex.getErrorCode());
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
index 64850dc194..0c2cebe1e3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java
@@ -1,6 +1,9 @@
package io.github.hectorvent.floci.services.redshift;
+import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerHandle;
@@ -11,6 +14,7 @@ import io.github.hectorvent.floci.services.redshift.model.ClusterSubnetGroup;
import io.github.hectorvent.floci.services.redshift.model.Endpoint;
import io.github.hectorvent.floci.services.redshift.model.Parameter;
import io.github.hectorvent.floci.services.redshift.model.Snapshot;
+import io.github.hectorvent.floci.services.redshift.proxy.RedshiftProxyManager;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
@@ -39,9 +43,9 @@ class RedshiftServiceTest {
private AccountAwareStorageBackend<ClusterParameterGroup> parameterGroupBackend;
private AccountAwareStorageBackend<ClusterSubnetGroup> subnetGroupBackend;
private RedshiftContainerManager cm;
- private io.github.hectorvent.floci.core.common.RegionResolver regionResolver;
- private io.github.hectorvent.floci.services.redshift.proxy.RedshiftProxyManager proxyManager;
- private io.github.hectorvent.floci.core.common.docker.DockerHostResolver dockerHostResolver;
+ private RegionResolver regionResolver;
+ private RedshiftProxyManager proxyManager;
+ private DockerHostResolver dockerHostResolver;
private RedshiftService service;
@BeforeEach
@@ -54,19 +58,19 @@ class RedshiftServiceTest {
parameterGroupBackend = mock(AccountAwareStorageBackend.class);
subnetGroupBackend = mock(AccountAwareStorageBackend.class);
cm = mock(RedshiftContainerManager.class);
- proxyManager = mock(io.github.hectorvent.floci.services.redshift.proxy.RedshiftProxyManager.class);
- dockerHostResolver = mock(io.github.hectorvent.floci.core.common.docker.DockerHostResolver.class);
+ proxyManager = mock(RedshiftProxyManager.class);
+ dockerHostResolver = mock(DockerHostResolver.class);
when(dockerHostResolver.resolve()).thenReturn("localhost");
- io.github.hectorvent.floci.config.EmulatorConfig config = mock(io.github.hectorvent.floci.config.EmulatorConfig.class);
- io.github.hectorvent.floci.config.EmulatorConfig.StorageConfig storageConfig = mock(io.github.hectorvent.floci.config.EmulatorConfig.StorageConfig.class);
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ EmulatorConfig.StorageConfig storageConfig = mock(EmulatorConfig.StorageConfig.class);
when(config.storage()).thenReturn(storageConfig);
when(storageConfig.persistentPath()).thenReturn("target/test-data");
- io.github.hectorvent.floci.config.EmulatorConfig.ServicesConfig servicesConfig =
- mock(io.github.hectorvent.floci.config.EmulatorConfig.ServicesConfig.class);
- io.github.hectorvent.floci.config.EmulatorConfig.RedshiftServiceConfig redshiftConfig =
- mock(io.github.hectorvent.floci.config.EmulatorConfig.RedshiftServiceConfig.class);
+ EmulatorConfig.ServicesConfig servicesConfig =
+ mock(EmulatorConfig.ServicesConfig.class);
+ EmulatorConfig.RedshiftServiceConfig redshiftConfig =
+ mock(EmulatorConfig.RedshiftServiceConfig.class);
when(config.services()).thenReturn(servicesConfig);
when(servicesConfig.redshift()).thenReturn(redshiftConfig);
when(redshiftConfig.proxyBasePort()).thenReturn(7100);
@@ -79,7 +83,7 @@ class RedshiftServiceTest {
when(sf.<ClusterSubnetGroup>create(eq("redshift"), eq("redshift-subnet-groups.json"), any())).thenReturn(subnetGroupBackend);
when(clusterBackend.accountId()).thenReturn("111111111111");
- regionResolver = new io.github.hectorvent.floci.core.common.RegionResolver("us-east-1", "111111111111");
+ regionResolver = new RegionResolver("us-east-1", "111111111111");
service = new RedshiftService(sf, cm, config, regionResolver, proxyManager, dockerHostResolver);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java
index e6a8cb3c72..a8d02b3020 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java
@@ -6,14 +6,21 @@ import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
class CopyStatementParserTest {
+ /** parse() returns S3Statement now; every COPY test wants the S3CopyFrom view. */
+ private static CopyStatementParser.S3CopyFrom copyFrom(String sql) {
+ CopyStatementParser.S3Statement s = CopyStatementParser.parse(sql);
+ return (CopyStatementParser.S3CopyFrom) s;
+ }
+
@Test
void parsesMinimalCopyFromKey() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(
+ CopyStatementParser.S3CopyFrom c = copyFrom(
"COPY sales FROM 's3://warehouse/data/sales.txt'");
assertEquals("sales", c.targetTable());
assertEquals(List.of(), c.columns());
@@ -28,7 +35,7 @@ class CopyStatementParserTest {
@Test
void parsesColumnsOptionsAndPrefix() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(
+ CopyStatementParser.S3CopyFrom c = copyFrom(
"COPY public.events (id, ts, note) FROM 's3://bkt/evt/' "
+ "GZIP DELIMITER ',' IGNOREHEADER 2 NULL AS '\\\\N' FORMAT AS CSV");
assertEquals("public.events", c.targetTable());
@@ -44,7 +51,7 @@ class CopyStatementParserTest {
@Test
void defaultsDelimiterToCommaForCsvAndTreatsHeaderKeywordAsOneLine() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(
+ CopyStatementParser.S3CopyFrom c = copyFrom(
"COPY t FROM 's3://b/k' CSV HEADER");
assertEquals(",", c.delimiter());
assertEquals(1, c.headerLines());
@@ -52,21 +59,21 @@ class CopyStatementParserTest {
@Test
void decodesTabDelimiterToken() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(
+ CopyStatementParser.S3CopyFrom c = copyFrom(
"COPY t FROM 's3://b/k' DELIMITER '\\\\t'");
assertEquals("\t", c.delimiter());
}
@Test
void bucketOnlyPathGivesEmptyPrefix() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse("COPY t FROM 's3://only-bucket'");
+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://only-bucket'");
assertEquals("only-bucket", c.bucket());
assertEquals("", c.keyOrPrefix());
}
@Test
void stripsLeadingComments() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(
+ CopyStatementParser.S3CopyFrom c = copyFrom(
"-- load nightly\n/* batch */ COPY t FROM 's3://b/k'");
assertEquals("t", c.targetTable());
}
@@ -95,7 +102,7 @@ class CopyStatementParserTest {
@Test
void quotedIdentifiersSurvive() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(
+ CopyStatementParser.S3CopyFrom c = copyFrom(
"COPY \"My Schema\".\"Tab\" (\"col one\") FROM 's3://b/k'");
assertEquals("\"My Schema\".\"Tab\"", c.targetTable());
assertEquals(List.of("\"col one\""), c.columns());
@@ -103,7 +110,7 @@ class CopyStatementParserTest {
@Test
void explicitDelimiterOverridesCsvDefault() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(
+ CopyStatementParser.S3CopyFrom c = copyFrom(
"COPY t FROM 's3://b/k' CSV DELIMITER '\t'");
assertEquals("\t", c.delimiter());
assertTrue(c.csv());
@@ -127,13 +134,13 @@ class CopyStatementParserTest {
@Test
void toleratesALoneTrailingSemicolon() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse("COPY t FROM 's3://b/k' GZIP;");
+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://b/k' GZIP;");
assertTrue(c.gzip());
}
@Test
void aQuotedKeywordInAnOptionValueDoesNotChangeParsing() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(
+ CopyStatementParser.S3CopyFrom c = copyFrom(
"COPY t FROM 's3://b/k' NULL AS 'csv' DELIMITER ';'");
assertFalse(c.csv());
assertEquals(";", c.delimiter());
@@ -164,8 +171,116 @@ class CopyStatementParserTest {
@Test
void parsesEscapedSingleQuoteInDelimiter() {
- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(
+ CopyStatementParser.S3CopyFrom c = copyFrom(
"COPY t FROM 's3://b/k' DELIMITER ''''");
assertEquals("'", c.delimiter());
}
+
+ private static CopyStatementParser.S3Unload unload(String sql) {
+ return (CopyStatementParser.S3Unload) CopyStatementParser.parse(sql);
+ }
+
+ @Test
+ void parsesMinimalUnload() {
+ CopyStatementParser.S3Unload u = unload(
+ "UNLOAD ('select id, name from sales') TO 's3://warehouse/out/'");
+ assertEquals("select id, name from sales", u.selectQuery());
+ assertEquals("warehouse", u.bucket());
+ assertEquals("out/", u.prefix());
+ assertEquals("|", u.delimiter());
+ assertFalse(u.csv());
+ assertFalse(u.gzip());
+ assertFalse(u.header());
+ assertFalse(u.manifest());
+ assertFalse(u.allowOverwrite());
+ assertTrue(u.parallel());
+ assertEquals(0L, u.maxFileSizeBytes());
+ assertNull(u.nullAs());
+ }
+
+ @Test
+ void parsesUnloadOptions() {
+ CopyStatementParser.S3Unload u = unload(
+ "UNLOAD ('select * from t') TO 's3://b/p/' "
+ + "FORMAT AS CSV DELIMITER ',' HEADER GZIP ADDQUOTES MANIFEST "
+ + "ALLOWOVERWRITE PARALLEL OFF NULL AS 'nil' MAXFILESIZE 5 MB");
+ assertTrue(u.csv());
+ assertEquals(",", u.delimiter());
+ assertTrue(u.header());
+ assertTrue(u.gzip());
+ assertTrue(u.addQuotes());
+ assertTrue(u.manifest());
+ assertTrue(u.allowOverwrite());
+ assertFalse(u.parallel());
+ assertEquals("nil", u.nullAs());
+ assertEquals(5L * 1024 * 1024, u.maxFileSizeBytes());
+ }
+
+ @Test
+ void unloadDefaultsCsvDelimiterToComma() {
+ assertEquals(",", unload("UNLOAD ('select 1') TO 's3://b/p/' CSV").delimiter());
+ }
+
+ @Test
+ void unloadParsesMaxFileSizeUnits() {
+ assertEquals(10L * 1024 * 1024, unload("UNLOAD ('select 1') TO 's3://b/p/' MAXFILESIZE 10 MB").maxFileSizeBytes());
+ assertEquals(1024L * 1024 * 1024 / 8, unload("UNLOAD ('select 1') TO 's3://b/p/' MAXFILESIZE 0.125 GB").maxFileSizeBytes());
+ assertEquals(4096L, unload("UNLOAD ('select 1') TO 's3://b/p/' MAXFILESIZE 4096").maxFileSizeBytes());
+ }
+
+ @Test
+ void unloadRejectsMaxFileSizeAboveTheBufferingCeiling() {
+ // The simulator buffers a file at a time, so a per-file size it cannot hold fails open.
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' MAXFILESIZE 2 GB"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' MAXFILESIZE 500 MB"));
+ }
+
+ @Test
+ void unloadUnescapesDoubledSingleQuotesInSelect() {
+ assertEquals("select 'x' from t",
+ unload("UNLOAD ('select ''x'' from t') TO 's3://b/p/'").selectQuery());
+ }
+
+ @Test
+ void unloadAcceptsWithCteSelect() {
+ assertNotNull(unload("UNLOAD ('with c as (select 1) select * from c') TO 's3://b/p/'"));
+ }
+
+ @Test
+ void unloadRejectsSubqueryBreakoutAttempts() {
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1) TO PROGRAM ''id'' --') TO 's3://b/p/'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1; drop table t') TO 's3://b/p/'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select $$x$$') TO 's3://b/p/'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1)') TO 's3://b/p/'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('delete from t') TO 's3://b/p/'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1 /* c */') TO 's3://b/p/'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select E''breakout''') TO 's3://b/p/'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select U&''breakout''') TO 's3://b/p/'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select \\1') TO 's3://b/p/'"));
+ }
+
+ @Test
+ void unloadRejectsUnsupportedOptions() {
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' PARQUET"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' ENCRYPTED"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' IAM_ROLE 'arn:aws:iam::0:role/r'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' REGION 'us-west-2'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' EXTENSION 'csv'"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' ZSTD"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' PARTITION BY (dt)"));
+ }
+
+ @Test
+ void unloadRejectsDuplicateOptionsAndUnknownTokens() {
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' GZIP GZIP"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' FOO"));
+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/'; SELECT 1"));
+ }
+
+ @Test
+ void unloadQuotedKeywordInValueDoesNotFlipParsing() {
+ CopyStatementParser.S3Unload u = unload("UNLOAD ('select 1') TO 's3://b/p/' NULL AS 'csv'");
+ assertFalse(u.csv());
+ assertEquals("csv", u.nullAs());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java
index a0f396f648..170dd4528c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java
@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.services.acm.CertificateGenerator;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;
import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;
+import io.github.hectorvent.floci.services.s3.S3Service;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -12,6 +13,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
@@ -68,7 +70,7 @@ class RedshiftAuthProxyTest {
proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),
"admin", "Secret123", "dev",
mock(RdsSigV4Validator.class), realTls(), (user, pw) -> true,
- mock(io.github.hectorvent.floci.services.s3.S3Service.class));
+ mock(S3Service.class));
proxy.start(proxyPort);
try (Socket client = new Socket("localhost", proxyPort)) {
@@ -122,7 +124,7 @@ class RedshiftAuthProxyTest {
proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),
"admin", "Secret123", "dev",
mock(RdsSigV4Validator.class), realTls(), (user, pw) -> true,
- mock(io.github.hectorvent.floci.services.s3.S3Service.class));
+ mock(S3Service.class));
proxy.start(proxyPort);
// Connect, then drop without ever sending a startup packet.
@@ -141,7 +143,7 @@ class RedshiftAuthProxyTest {
// yet; free it shortly after so the retrying bind can finally take it.
ServerSocket squatter = new ServerSocket();
squatter.setReuseAddress(true);
- squatter.bind(new java.net.InetSocketAddress(proxyPort));
+ squatter.bind(new InetSocketAddress(proxyPort));
Thread.ofVirtual().start(() -> {
try {
Thread.sleep(200);
@@ -153,7 +155,7 @@ class RedshiftAuthProxyTest {
proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),
"admin", "Secret123", "dev",
mock(RdsSigV4Validator.class), realTls(), (user, pw) -> true,
- mock(io.github.hectorvent.floci.services.s3.S3Service.class));
+ mock(S3Service.class));
proxy.start(proxyPort); // must not throw despite the port being busy at first
assertTrue(portAccepts(proxyPort), "proxy never bound the port after the squatter released it");
@@ -166,7 +168,7 @@ class RedshiftAuthProxyTest {
proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),
"admin", "old", "dev",
mock(RdsSigV4Validator.class), realTls(), (user, pw) -> true,
- mock(io.github.hectorvent.floci.services.s3.S3Service.class));
+ mock(S3Service.class));
proxy.start(proxyPort);
proxy.updateMasterPassword("rotated");
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
index a7fdf5c879..2ab9e7b92e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
@@ -12,6 +12,7 @@ import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
+import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
@@ -212,4 +213,52 @@ class RedshiftInterceptingBridgeTest {
assertEquals('Q', forwarded.type());
assertTrue(forwarded.getSql().contains("s3://wh/k"), forwarded.getSql());
}
+
+ @Test
+ void unloadQueryIsDispatchedToTheSimulator() throws Exception {
+ startBridge();
+ Mockito.when(s3Stub.listObjectsWithPrefixes(Mockito.eq("b"), Mockito.eq("out/"),
+ Mockito.isNull(), Mockito.anyInt(), Mockito.any(), Mockito.any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+ AtomicReference<byte[]> put = new AtomicReference<>();
+ Mockito.when(s3Stub.putObject(Mockito.eq("b"), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
+ .thenAnswer(inv -> {
+ put.compareAndSet(null, inv.getArgument(2));
+ return null;
+ });
+
+ // Fake backend: expect the fabricated COPY (...) TO STDOUT and play one row.
+ Thread be = Thread.ofVirtual().start(() -> {
+ try {
+ PostgresWireDecoder in = new PostgresWireDecoder(testBackendEnd.getInputStream());
+ PostgresWireDecoder.FrontendMessage q = in.nextMessage();
+ assertTrue(q.getSql().contains("TO STDOUT"), q.getSql());
+ OutputStream out = testBackendEnd.getOutputStream();
+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});
+ byte[] row = "1|a\n".getBytes(StandardCharsets.US_ASCII);
+ out.write('d');
+ out.write(new byte[]{(byte) (0), 0, 0, (byte) (4 + row.length)});
+ out.write(row);
+ out.write(new byte[]{'c', 0, 0, 0, 4});
+ byte[] tag = "COPY 1\0".getBytes(StandardCharsets.US_ASCII);
+ out.write('C');
+ out.write(new byte[]{0, 0, 0, (byte) (4 + tag.length)});
+ out.write(tag);
+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});
+ out.flush();
+ } catch (IOException e) {
+ LOG.debugv(e, "fake backend ended");
+ }
+ });
+
+ OutputStream clientOut = testClientEnd.getOutputStream();
+ clientOut.write(PostgresWireDecoder.encodeQuery("UNLOAD ('select a,b from t') TO 's3://b/out/'"));
+ clientOut.flush();
+
+ PostgresWireDecoder fromPump = new PostgresWireDecoder(testClientEnd.getInputStream());
+ assertEquals('C', fromPump.nextMessage().type());
+ assertEquals('Z', fromPump.nextMessage().type());
+ be.join();
+ assertNotNull(put.get(), "the simulator should have written one S3 object");
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java
index d62c6658f8..dcbc75bd2c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.redshift.proxy;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;
import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;
+import io.github.hectorvent.floci.services.s3.S3Service;
import org.junit.jupiter.api.Test;
import java.io.IOException;
@@ -28,7 +29,7 @@ class RedshiftProxyManagerTest {
private RedshiftProxyManager newManager() {
return new RedshiftProxyManager(
mock(RdsSigV4Validator.class), mock(RdsProxyTlsCertificates.class),
- mock(io.github.hectorvent.floci.services.s3.S3Service.class));
+ mock(S3Service.class));
}
private static void start(RedshiftProxyManager manager, String key, int proxyPort) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java
index 28b9a8aeaa..9570a4158d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java
@@ -7,6 +7,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -14,12 +15,17 @@ import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -108,6 +114,598 @@ class S3CopySimulatorTest {
return new PostgresWireDecoder(testBackend.getInputStream()).nextMessage();
}
+ /** Play a PostgreSQL backend for a fabricated COPY (...) TO STDOUT: 'H', N 'd' rows, 'c', 'C', 'Z'. */
+ private void playUnloadBackend(String... rows) throws IOException {
+ OutputStream out = testBackend.getOutputStream();
+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());
+ PostgresWireDecoder.FrontendMessage q = in.nextMessage();
+ assertEquals('Q', q.type());
+ assertTrue(q.getSql().contains("TO STDOUT"), q.getSql());
+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0}); // CopyOutResponse
+ for (String row : rows) {
+ byte[] b = row.getBytes(StandardCharsets.US_ASCII);
+ out.write('d');
+ out.write(intBytes(4 + b.length));
+ out.write(b);
+ }
+ out.write(new byte[]{'c', 0, 0, 0, 4}); // CopyDone
+ byte[] tag = "COPY 2\0".getBytes(StandardCharsets.US_ASCII);
+ out.write('C');
+ out.write(intBytes(4 + tag.length));
+ out.write(tag);
+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});
+ out.flush();
+ }
+
+ private CopyStatementParser.S3Unload unloadSpec(String bucket, String prefix,
+ boolean gzip, boolean manifest, boolean allowOverwrite, boolean parallel, long maxFileSize) {
+ return new CopyStatementParser.S3Unload("select a,b from t", bucket, prefix,
+ "|", false, gzip, false, false, null, manifest, allowOverwrite, parallel, maxFileSize);
+ }
+
+ @Test
+ void unloadWritesSingleObjectAndForwardsCommandComplete() throws Exception {
+ Map<String, byte[]> written = new ConcurrentHashMap<>();
+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {
+ written.put(inv.getArgument(1), inv.getArgument(2));
+ return null;
+ });
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
+ Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n"));
+
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ assertTrue(handled);
+ assertEquals(1, written.size());
+ assertTrue(written.containsKey("out/0000_part_00"), written.keySet().toString());
+ assertEquals("1|alice\n2|bob\n", new String(written.get("out/0000_part_00"), StandardCharsets.US_ASCII));
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ assertEquals('C', in.nextMessage().type());
+ assertEquals('Z', in.nextMessage().type());
+ }
+
+ @Test
+ void unloadRepeatsTheHeaderRowInEverySlice() throws Exception {
+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 4; // one data row per slice
+ try {
+ Map<String, byte[]> written = new ConcurrentHashMap<>();
+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {
+ written.put(inv.getArgument(1), inv.getArgument(2));
+ return null;
+ });
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+
+ CopyStatementParser.S3Unload spec = new CopyStatementParser.S3Unload(
+ "select a,b from t", "wh", "out/", "|", true, false, true, false, null,
+ false, false, true, 0);
+ // The fake backend stands in for PostgreSQL: the first CopyData frame is the header row.
+ Thread backend = backendThread(() -> playUnloadBackend("h1|h2\n", "1|a\n", "2|b\n", "3|c\n"));
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ assertEquals(3, written.size(), written.keySet().toString());
+ for (Map.Entry<String, byte[]> e : written.entrySet()) {
+ String body = new String(e.getValue(), StandardCharsets.US_ASCII);
+ assertTrue(body.startsWith("h1|h2\n"), e.getKey() + " -> " + body);
+ }
+ } finally {
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;
+ }
+ }
+
+ @Test
+ void unloadSplitsIntoMultipleObjectsOnNewlineBoundaries() throws Exception {
+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 6; // force a split after each row
+ try {
+ Map<String, byte[]> written = new ConcurrentHashMap<>();
+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {
+ written.put(inv.getArgument(1), inv.getArgument(2));
+ return null;
+ });
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
+ Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n", "3|carol\n"));
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ assertTrue(written.containsKey("out/0000_part_00"));
+ assertTrue(written.containsKey("out/0001_part_00"));
+ assertTrue(written.containsKey("out/0002_part_00"));
+ assertEquals(3, written.size(), written.keySet().toString());
+ StringBuilder all = new StringBuilder();
+ written.entrySet().stream().sorted(Map.Entry.comparingByKey())
+ .forEach(e -> all.append(new String(e.getValue(), StandardCharsets.US_ASCII)));
+ assertEquals("1|alice\n2|bob\n3|carol\n", all.toString());
+ } finally {
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;
+ }
+ }
+
+ @Test
+ void unloadParallelOffUsesThreeDigitKeyNames() throws Exception {
+ Map<String, byte[]> written = new ConcurrentHashMap<>();
+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {
+ written.put(inv.getArgument(1), inv.getArgument(2));
+ return null;
+ });
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, false, 0);
+ Thread backend = backendThread(() -> playUnloadBackend("1|a\n"));
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ assertTrue(written.containsKey("out/000"), written.keySet().toString());
+ }
+
+ @Test
+ void unloadGzipAppendsGzSuffixAndWritesDecompressibleBytes() throws Exception {
+ Map<String, byte[]> written = new ConcurrentHashMap<>();
+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {
+ written.put(inv.getArgument(1), inv.getArgument(2));
+ return null;
+ });
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", true, false, false, true, 0);
+ Thread backend = backendThread(() -> playUnloadBackend("1|a\n", "2|b\n"));
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ assertTrue(written.containsKey("out/0000_part_00.gz"), written.keySet().toString());
+ byte[] gz = written.get("out/0000_part_00.gz");
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(gz))) {
+ in.transferTo(out);
+ }
+ assertEquals("1|a\n2|b\n", out.toString(StandardCharsets.US_ASCII));
+ }
+
+ @Test
+ void unloadManifestListsEveryDataObject() throws Exception {
+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 8;
+ try {
+ Map<String, byte[]> written = new ConcurrentHashMap<>();
+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {
+ written.put(inv.getArgument(1), inv.getArgument(2));
+ return null;
+ });
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, false, true, 0);
+ Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n", "3|carol\n"));
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ String manifest = new String(written.get("out/manifest"), StandardCharsets.UTF_8);
+ assertTrue(manifest.contains("s3://wh/out/0000_part_00"), manifest);
+ assertTrue(manifest.contains("s3://wh/out/0001_part_00"), manifest);
+ assertTrue(manifest.contains("\"content_length\":"), manifest);
+ } finally {
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;
+ }
+ }
+
+ @Test
+ void unloadIntoNonEmptyPrefixWithoutAllowOverwriteErrorsAndSkipsSelect() throws Exception {
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(
+ List.of(new S3Object("wh", "out/old", new byte[]{1}, "text/plain")), List.of(), false, null));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
+ // No backend thread: the select must never run.
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ assertTrue(handled);
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ assertEquals('E', in.nextMessage().type());
+ assertEquals('Z', in.nextMessage().type());
+ }
+
+ @Test
+ void unloadForwardsBackendErrorWhenSelectIsInvalid() throws Exception {
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
+
+ Thread backend = backendThread(() -> {
+ OutputStream out = testBackend.getOutputStream();
+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());
+ assertEquals('Q', in.nextMessage().type());
+ byte[] err = "SERROR\0C42703\0Mcolumn \"nope\" does not exist\0\0".getBytes(StandardCharsets.US_ASCII);
+ out.write('E');
+ out.write(intBytes(4 + err.length));
+ out.write(err);
+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});
+ out.flush();
+ });
+
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ assertEquals('E', in.nextMessage().type());
+ assertEquals('Z', in.nextMessage().type());
+ testClient.setSoTimeout(400);
+ assertThrows(SocketTimeoutException.class, () -> testClient.getInputStream().read());
+ }
+
+ @Test
+ void unloadAbortsWhenResultExceedsTotalCeiling() throws Exception {
+ long saved = S3CopySimulator.UNLOAD_MAX_TOTAL_BYTES;
+ S3CopySimulator.UNLOAD_MAX_TOTAL_BYTES = 16;
+ try {
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 1_000_000);
+
+ Thread backend = backendThread(() -> {
+ OutputStream out = testBackend.getOutputStream();
+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());
+ assertEquals('Q', in.nextMessage().type());
+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});
+ byte[] big = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n".getBytes(StandardCharsets.US_ASCII);
+ out.write('d');
+ out.write(intBytes(4 + big.length));
+ out.write(big);
+ out.flush();
+ try {
+ out.write(new byte[]{'c', 0, 0, 0, 4});
+ out.flush();
+ } catch (IOException ignored) {
+ // Backend socket is closed immediately by simulator on ceiling exceed
+ }
+ });
+
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ PostgresWireDecoder.FrontendMessage err = in.nextMessage();
+ assertEquals('E', err.type());
+ assertTrue(new String(err.body(), StandardCharsets.US_ASCII).contains("54000"));
+ assertEquals('Z', in.nextMessage().type());
+ assertNull(in.nextMessage(), "Client must receive EOF after ReadyForQuery on connection close");
+ } finally {
+ S3CopySimulator.UNLOAD_MAX_TOTAL_BYTES = saved;
+ }
+ }
+
+ @Test
+ void unloadInTransactionBlockReportsFailedTransactionStatusOnError() throws Exception {
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(
+ List.of(new S3Object("wh", "out/old", new byte[]{1}, "text/plain")), List.of(), false, null));
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
+
+ Thread backend = backendThread(() -> {
+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());
+ PostgresWireDecoder.FrontendMessage q = in.nextMessage();
+ assertEquals('Q', q.type());
+ assertTrue(q.getSql().contains("FLOCI_ABORT_TX"));
+ OutputStream out = testBackend.getOutputStream();
+ byte[] err = "SERROR\0C42601\0Msyntax error\0\0".getBytes(StandardCharsets.US_ASCII);
+ out.write('E');
+ out.write(intBytes(4 + err.length));
+ out.write(err);
+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'E'});
+ out.flush();
+ });
+
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'T');
+ joinBackend(backend);
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ assertEquals('E', in.nextMessage().type());
+ PostgresWireDecoder.FrontendMessage z = in.nextMessage();
+ assertEquals('Z', z.type());
+ assertEquals('E', (char) z.body()[0]);
+ }
+
+ @Test
+ void unloadStreamingFailureDrainsBackendAndSendsSingleError() throws Exception {
+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 8; // Force flush on first row
+ try {
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+ when(s3.putObject(eq("wh"), any(), any(), any(), any()))
+ .thenThrow(new RuntimeException("simulated S3 storage failure"));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
+
+ Thread backend = backendThread(() -> {
+ OutputStream out = testBackend.getOutputStream();
+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());
+ assertEquals('Q', in.nextMessage().type());
+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});
+ // Send multiple rows so the backend has pending messages when putObject throws mid-stream on the first slice
+ byte[] row1 = "1|alice\n".getBytes(StandardCharsets.US_ASCII);
+ out.write('d');
+ out.write(intBytes(4 + row1.length));
+ out.write(row1);
+ byte[] row2 = "2|bob\n".getBytes(StandardCharsets.US_ASCII);
+ out.write('d');
+ out.write(intBytes(4 + row2.length));
+ out.write(row2);
+ out.write(new byte[]{'c', 0, 0, 0, 4});
+ byte[] tag = "COPY 2\0".getBytes(StandardCharsets.US_ASCII);
+ out.write('C');
+ out.write(intBytes(4 + tag.length));
+ out.write(tag);
+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});
+ out.flush();
+ });
+
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ assertTrue(handled);
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ assertEquals('E', in.nextMessage().type());
+ assertEquals('Z', in.nextMessage().type());
+ testClient.setSoTimeout(400);
+ assertThrows(SocketTimeoutException.class, () -> testClient.getInputStream().read(),
+ "no leftover or duplicate messages should reach client");
+ } finally {
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;
+ }
+ }
+
+ @Test
+ void unloadAbortsWhenMemoryBudgetExhaustedReports53400() throws Exception {
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
+
+ S3CopySimulator.UNLOAD_HEAP_MIB.acquire(192);
+ try {
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ assertTrue(handled);
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ PostgresWireDecoder.FrontendMessage err = in.nextMessage();
+ assertEquals('E', err.type());
+ String errBody = new String(err.body(), StandardCharsets.UTF_8);
+ assertTrue(errBody.contains("53400"), errBody);
+ assertTrue(errBody.contains("memory budget exhausted"), errBody);
+ assertEquals('Z', in.nextMessage().type());
+ } finally {
+ S3CopySimulator.UNLOAD_HEAP_MIB.release(192);
+ }
+ }
+
+ @Test
+ void unloadFabricatedCopySqlMatchesFormattingOptions() throws Exception {
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+ CopyStatementParser.S3Unload spec = new CopyStatementParser.S3Unload(
+ "select a from t", "wh", "out/", "\t", true, false, false, true, "\\N", false, true, true, 0);
+
+ AtomicReference<String> seenSql = new AtomicReference<>();
+ Thread backend = backendThread(() -> {
+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());
+ PostgresWireDecoder.FrontendMessage q = in.nextMessage();
+ seenSql.set(q.getSql());
+ OutputStream out = testBackend.getOutputStream();
+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});
+ out.write(new byte[]{'c', 0, 0, 0, 4});
+ byte[] tag = "COPY 0\0".getBytes(StandardCharsets.US_ASCII);
+ out.write('C');
+ out.write(intBytes(4 + tag.length));
+ out.write(tag);
+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});
+ out.flush();
+ });
+
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ String sql = seenSql.get();
+ assertNotNull(sql);
+ assertTrue(sql.contains("FORMAT csv"), sql);
+ assertTrue(sql.contains("DELIMITER '\t'"), sql);
+ assertTrue(sql.contains("HEADER true"), sql);
+ assertTrue(sql.contains("FORCE_QUOTE *"), sql);
+ assertTrue(sql.contains("NULL '\\N'"), sql);
+ }
+
+ @Test
+ void unloadZeroRowsEmitsSingleEmptyObject() throws Exception {
+ Map<String, byte[]> written = new ConcurrentHashMap<>();
+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {
+ written.put(inv.getArgument(1), inv.getArgument(2));
+ return null;
+ });
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
+ Thread backend = backendThread(() -> {
+ OutputStream out = testBackend.getOutputStream();
+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());
+ assertEquals('Q', in.nextMessage().type());
+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});
+ out.write(new byte[]{'c', 0, 0, 0, 4});
+ byte[] tag = "COPY 0\0".getBytes(StandardCharsets.US_ASCII);
+ out.write('C');
+ out.write(intBytes(4 + tag.length));
+ out.write(tag);
+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});
+ out.flush();
+ });
+
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ assertTrue(handled);
+ assertEquals(1, written.size());
+ assertTrue(written.containsKey("out/0000_part_00"));
+ assertEquals(0, written.get("out/0000_part_00").length);
+ }
+
+ @Test
+ void unloadBucketNotFoundSurfacesNotFoundMessageInsteadOfAccessDenied() throws Exception {
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenThrow(new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ assertTrue(handled);
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ PostgresWireDecoder.FrontendMessage err = in.nextMessage();
+ assertEquals('E', err.type());
+ String body = new String(err.body(), StandardCharsets.UTF_8);
+ assertTrue(body.contains("not found"), body);
+ assertFalse(body.contains("access denied"), body);
+ assertEquals('Z', in.nextMessage().type());
+ }
+
+ @Test
+ void unloadManifestFailureCleansUpWrittenDataObjects() throws Exception {
+ List<String> deleted = new ArrayList<>();
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {
+ String key = inv.getArgument(1);
+ if (key.endsWith("manifest")) {
+ throw new RuntimeException("disk full writing manifest");
+ }
+ return null;
+ });
+ when(s3.deleteObject(eq("wh"), any())).thenAnswer(inv -> {
+ deleted.add(inv.getArgument(1));
+ return null;
+ });
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, false, true, 0);
+ Thread backend = backendThread(() -> playUnloadBackend("1|a\n"));
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ assertEquals(1, deleted.size());
+ assertEquals("out/0000_part_00", deleted.get(0));
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ assertEquals('E', in.nextMessage().type());
+ assertEquals('Z', in.nextMessage().type());
+ }
+
+ @Test
+ void unloadManifestFailureUnderAllowOverwriteLeavesEveryDataObjectInPlace() throws Exception {
+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 4; // one data row per slice
+ try {
+ List<String> deleted = new ArrayList<>();
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {
+ if (((String) inv.getArgument(1)).endsWith("manifest")) {
+ throw new RuntimeException("disk full writing manifest");
+ }
+ return null;
+ });
+ when(s3.deleteObject(eq("wh"), any())).thenAnswer(inv -> {
+ deleted.add(inv.getArgument(1));
+ return null;
+ });
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, true, true, 0);
+ Thread backend = backendThread(() -> playUnloadBackend("1|a\n", "2|b\n"));
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ // Under ALLOWOVERWRITE a written key may have replaced pre-existing (or another
+ // operation's) data, so cleanup deletes nothing and the partial result is left behind.
+ assertTrue(deleted.isEmpty(), "ALLOWOVERWRITE cleanup must not delete any object, got " + deleted);
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ assertEquals('E', in.nextMessage().type());
+ assertEquals('Z', in.nextMessage().type());
+ } finally {
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;
+ }
+ }
+
+ @Test
+ void unloadWithoutAllowOverwriteFailsIfListBucketDenied() throws Exception {
+ doThrow(new AwsException("AccessDenied", "Access Denied", 403))
+ .when(s3).authorizeAnonymousListBucket(eq("wh"));
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);
+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ assertTrue(handled);
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ PostgresWireDecoder.FrontendMessage err = in.nextMessage();
+ assertEquals('E', err.type());
+ String body = new String(err.body(), StandardCharsets.UTF_8);
+ assertTrue(body.contains("42501"), body);
+ assertTrue(body.contains("access denied"), body);
+ assertEquals('Z', in.nextMessage().type());
+ }
+
+ @Test
+ void unloadBackendEofMidStreamWithManifestCleansUpWrittenDataObjects() throws Exception {
+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 6;
+ try {
+ List<String> deleted = new ArrayList<>();
+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))
+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
+ when(s3.deleteObject(eq("wh"), any())).thenAnswer(inv -> {
+ deleted.add(inv.getArgument(1));
+ return null;
+ });
+
+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, false, true, 0);
+
+ Thread backend = backendThread(() -> {
+ OutputStream out = testBackend.getOutputStream();
+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());
+ assertEquals('Q', in.nextMessage().type());
+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});
+ byte[] row = "1|alice\n".getBytes(StandardCharsets.US_ASCII);
+ out.write('d');
+ out.write(intBytes(4 + row.length));
+ out.write(row);
+ out.flush();
+ testBackend.close();
+ });
+
+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');
+ joinBackend(backend);
+
+ assertEquals(1, deleted.size());
+ assertEquals("out/0000_part_00", deleted.get(0));
+
+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());
+ PostgresWireDecoder.FrontendMessage err = in.nextMessage();
+ assertEquals('E', err.type());
+ String body = new String(err.body(), StandardCharsets.UTF_8);
+ assertTrue(body.contains("backend closed mid-stream"), body);
+ assertEquals('Z', in.nextMessage().type());
+ } finally {
+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;
+ }
+ }
+
/** Play a minimal happy-path PostgreSQL backend: 'G' then, after CopyDone, 'C' and 'Z'. */
private void playHappyBackend(ByteArrayOutputStream capturedCopyData) throws IOException {
OutputStream out = testBackend.getOutputStream();
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
index 77d3be2145..7d6d554b38 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
@@ -1149,4 +1149,18 @@ class S3ServiceTest {
"configuration config-" + i + " was lost by a concurrent put");
}
}
+
+ @Test
+ void authorizeAnonymousPutObjectIsANoOpWhenEnforceAuthIsOff() {
+ // Default test config has FLOCI_SERVICES_S3_ENFORCE_AUTH unset/false.
+ s3Service.createBucket("anon-put-bucket", "us-east-1");
+ assertDoesNotThrow(() -> s3Service.authorizeAnonymousPutObject("anon-put-bucket", "some/key"));
+ }
+
+ @Test
+ void authorizeAnonymousDeleteObjectIsANoOpWhenEnforceAuthIsOff() {
+ s3Service.createBucket("anon-del-bucket", "us-east-1");
+ assertDoesNotThrow(() -> s3Service.authorizeAnonymousDeleteObject("anon-del-bucket", "some/key"));
+ }
}
+