Light, fluffy, and always free - The AWS Local Emulator alternative
feat(redshift): emulate Redshift UNLOAD TO s3 over Simple Query protocol (#3129)
Vũ Thế Huy4570bf62c5f8
diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.javaindex 28fa758ee9..db816e09e1 100644--- a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java@@ -13,6 +13,16 @@ import software.amazon.awssdk.services.redshift.model.DescribeClustersRequest;import software.amazon.awssdk.services.redshift.model.DescribeClustersResponse;import software.amazon.awssdk.services.redshift.model.DeleteClusterRequest;import software.amazon.awssdk.services.redshift.model.DeleteClusterResponse;+import software.amazon.awssdk.services.s3.S3Client;+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;+import software.amazon.awssdk.services.s3.model.GetObjectRequest;+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;++import java.nio.charset.StandardCharsets;+import java.sql.Connection;+import java.sql.DriverManager;+import java.sql.Statement;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;@@ -59,6 +69,45 @@ public class RedshiftTest {@Test@Order(2)+ public void testUnloadOverSimpleQueryWritesToS3() throws Exception {+ RedshiftClient client = getClient();+ Cluster cluster = client.describeClusters(DescribeClustersRequest.builder()+ .clusterIdentifier("test-cluster")+ .build()).clusters().get(0);+ String jdbcUrl = "jdbc:postgresql://" + cluster.endpoint().address() + ":"+ + cluster.endpoint().port() + "/dev?preferQueryMode=simple";++ S3Client s3 = TestFixtures.s3Client();+ String bucket = "redshift-unload-compat";+ s3.createBucket(CreateBucketRequest.builder().bucket(bucket).build());++ try (Connection conn = DriverManager.getConnection(jdbcUrl, "admin", "Password123");+ Statement st = conn.createStatement()) {+ st.execute("DROP TABLE IF EXISTS unload_src");+ st.execute("CREATE TABLE unload_src (id int, name text)");+ st.execute("INSERT INTO unload_src VALUES (1, 'alice'), (2, 'bob')");+ st.execute("UNLOAD ('select id, name from unload_src order by id') "+ + "TO 's3://redshift-unload-compat/out/'");+ }++ ListObjectsV2Response listing = s3.listObjectsV2(ListObjectsV2Request.builder()+ .bucket(bucket)+ .prefix("out/")+ .build());+ assertTrue(listing.keyCount() >= 1, "UNLOAD must write at least one object");++ StringBuilder all = new StringBuilder();+ listing.contents().stream()+ .sorted((a, b) -> a.key().compareTo(b.key()))+ .forEach(obj -> all.append(new String(+ s3.getObjectAsBytes(GetObjectRequest.builder().bucket(bucket).key(obj.key()).build())+ .asByteArray(),+ StandardCharsets.UTF_8)));+ assertEquals("1|alice\n2|bob\n", all.toString());+ }++ @Test+ @Order(3)public void testDeleteCluster() {RedshiftClient client = getClient();DeleteClusterResponse res = client.deleteCluster(DeleteClusterRequest.builder()diff --git a/docs/services/redshift.md b/docs/services/redshift.mdindex 8d632a0085..958877cff3 100644--- a/docs/services/redshift.md+++ b/docs/services/redshift.md@@ -175,11 +175,45 @@ order) through its own S3 service and streams the rows into the backing PostgreS- The rewrite is textual (regex-based). It masks single-quoted string literals first, so `DEFAULT` / `CHECK` string values are safe, but it is **not** comment-aware and does not recognize escape strings (`E'...'`): an apostrophe inside a `--` or `/* */` comment can make the rewrite skip a Redshift clause. That fails safe: the statement then reaches PostgreSQL, which returns its own syntax error, but avoid apostrophes-in-comments in `CREATE TABLE` / `ALTER TABLE`.- A `rewrite` failure or any statement the interceptor does not recognize is forwarded unmodified (fail-open); PostgreSQL then rejects the Redshift-only syntax itself.- Simple Query ('Q') messages larger than 16 MiB bypass the interceptor and stream through verbatim without heap buffering; non-query traffic also streams through with no size limit.-- `UNLOAD (...) TO 's3://...'` is **not** yet emulated (planned).++### UNLOAD to S3++`UNLOAD ('<select-statement>') TO 's3://<bucket>/<prefix>' [options]` sent over the+Simple Query protocol runs the select on the backing PostgreSQL container and writes+the result to S3 as one or more objects under `<prefix>`.++- Framing defaults to pipe-delimited text; `FORMAT CSV` (or `CSV`) switches to CSV+ with a `,` default. `DELIMITER`, `HEADER`, `NULL AS`, and `ADDQUOTES` are honoured+ (`ADDQUOTES` and `HEADER` force CSV framing on PostgreSQL 15).+- `PARALLEL ON` (default) names objects `<prefix>0000_part_00`,+ `<prefix>0001_part_00`, and so on; `PARALLEL OFF` names them `<prefix>000`,+ `<prefix>001`, and so on. The emulator has a single backend node, so more than one+ object appears only when the result exceeds the per-file size. When `HEADER` is+ set, the header row is repeated at the top of every object.+- `MAXFILESIZE [AS] <n> [MB|GB]` sets the per-file size; a bare number is bytes.+ Each file is buffered in memory, so the default is 6 MiB (real Redshift defaults+ to 6.2 GB) and a whole UNLOAD result is capped at 256 MiB; a larger result fails+ with a SQL error (54000). A `MAXFILESIZE` above that 256 MiB cap is not+ intercepted at all: the statement is forwarded and PostgreSQL reports its own error.+- A zero-row result still writes one object (empty, or the header row alone when+ `HEADER` is set).+- `GZIP` compresses each object and appends `.gz` to its key.+- `MANIFEST` writes `<prefix>manifest` listing every object with its+ `content_length`.+- Without `ALLOWOVERWRITE`, a non-empty target prefix fails with SQL error XX000 and the select+ does not run. A failed UNLOAD then removes any objects it had already written. With+ `ALLOWOVERWRITE` a failed UNLOAD leaves its objects in place (they may have replaced prior data,+ so they are not deleted); a `MANIFEST` request that fails this way can leave data objects without+ a manifest, and rerunning the same statement overwrites them.+- S3 access is authorized as an unsigned request, like COPY from S3.+- Any other option (`PARQUET`, `ENCRYPTED`, `REGION`, `IAM_ROLE` / `CREDENTIALS`,+ `ZSTD`, `EXTENSION`, `CLEANPATH`, `PARTITION`, and so on) is not intercepted; the+ statement is forwarded and PostgreSQL reports its own error.+- Extended Query protocol UNLOAD (a JDBC `PreparedStatement`) is not intercepted.## Out of Scope-- Real Redshift SQL semantics: the data plane is stock PostgreSQL. Redshift-only table DDL keywords (DISTSTYLE / DISTKEY / SORTKEY / ENCODE) are stripped so CREATE TABLE / ALTER TABLE executes (see [SQL Interceptor](#sql-interceptor)), but the distribution/sort behavior they request is not; UNLOAD to S3 and SUPER/SPECTRUM are not emulated.+- Real Redshift SQL semantics: the data plane is stock PostgreSQL. Redshift-only table DDL keywords (DISTSTYLE / DISTKEY / SORTKEY / ENCODE) are stripped so CREATE TABLE / ALTER TABLE executes (see [SQL Interceptor](#sql-interceptor)), but the distribution/sort behavior they request is not; SUPER/SPECTRUM are not emulated.- Multi-node clusters: `NodeType` and `NumberOfNodes` are stored as metadata; every cluster is a single PostgreSQL container.- Parameter groups apply no real engine settings; values are stored and echoed back only.- Subnet groups, VPC routing, and security groups are metadata only.diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.javaindex 215d927975..c051efd71c 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/container/RedshiftContainerManager.java@@ -406,7 +406,7 @@ public class RedshiftContainerManager {lastOutput = e.getMessage();}try {- java.util.concurrent.TimeUnit.SECONDS.sleep(1);+ TimeUnit.SECONDS.sleep(1);} catch (InterruptedException e) {Thread.currentThread().interrupt();throw new IllegalStateException("Interrupted initializing " + description + " in " + containerName, e);diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.javaindex 3c990300f1..1fe4cb6eeb 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParser.java@@ -6,12 +6,16 @@ import java.util.regex.Matcher;import java.util.regex.Pattern;/**- * Recognises a Simple Query statement that is an S3 {@code COPY <table> FROM 's3://...'}.- * Any other statement, or a COPY that uses an option this simulator cannot honour, returns- * {@code null} so the bridge falls back to DDL rewriting and PostgreSQL reports its own error.+ * Recognises a Simple Query statement that is an S3 {@code COPY <table> FROM 's3://...'}+ * or {@code UNLOAD ('<select>') TO 's3://...'}. Any other statement, or a statement that uses+ * an option this simulator cannot honour, returns {@code null} so the bridge falls back to DDL+ * rewriting and PostgreSQL reports its own error.*/public final class CopyStatementParser {+ public sealed interface S3Statement permits S3CopyFrom, S3Unload {+ }+public record S3CopyFrom(String targetTable,List<String> columns,@@ -21,13 +25,52 @@ public final class CopyStatementParser {int headerLines,boolean gzip,boolean csv,- String nullAs) {+ String nullAs) implements S3Statement {+ }++ public record S3Unload(+ String selectQuery,+ String bucket,+ String prefix,+ String delimiter,+ boolean header,+ boolean gzip,+ boolean csv,+ boolean addQuotes,+ String nullAs,+ boolean manifest,+ boolean allowOverwrite,+ boolean parallel,+ long maxFileSizeBytes) implements S3Statement {}private static final Pattern COPY_PATTERN = Pattern.compile("(?is)^\\s*COPY\\s+((?:\"[^\"]*\"|[^\\s(])+)\\s*(?:\\(([^)]*)\\))?\\s+FROM\\s*"+ "['\"]s3://([^/'\"\\s]+)(?:/([^'\"\\s]*))?['\"]\\s*(.*)$");+ private static final Pattern UNLOAD_PATTERN = Pattern.compile(+ "(?is)^\\s*UNLOAD\\s*\\(\\s*'(.*)'\\s*\\)\\s*TO\\s*"+ + "['\"]s3://([^/'\"\\s]+)(?:/([^'\"\\s]*))?['\"]\\s*(.*)$");++ private static final Pattern PARALLEL_PATTERN = Pattern.compile(+ "(?i)\\bPARALLEL\\b(?:\\s+(ON|OFF|TRUE|FALSE))?");+ private static final Pattern MAXFILESIZE_PATTERN = Pattern.compile(+ "(?i)\\bMAXFILESIZE\\s+(?:AS\\s+)?(\\d+(?:\\.\\d+)?)\\s*(MB|GB)?\\b");+ private static final Pattern ADDQUOTES_PATTERN = Pattern.compile("(?i)\\bADDQUOTES\\b");+ private static final Pattern MANIFEST_PATTERN = Pattern.compile("(?i)\\bMANIFEST\\b");+ private static final Pattern ALLOWOVERWRITE_PATTERN = Pattern.compile("(?i)\\bALLOWOVERWRITE\\b");++ /**+ * UNLOAD options this simulator cannot honour. Distinct from the COPY set:+ * MANIFEST / ALLOWOVERWRITE / PARALLEL / MAXFILESIZE are UNLOAD-supported here,+ * while EXTENSION / CLEANPATH / PARTITION are UNLOAD-only and unsupported.+ */+ private static final Pattern UNLOAD_UNSUPPORTED_CLAUSE = Pattern.compile(+ "(?i)\\b(FIXEDWIDTH|PARQUET|AVRO|ORC|JSON|SHAPEFILE|BZIP2|LZOP|ZSTD"+ + "|ENCRYPTED|ENCODING|REGION|CREDENTIALS|IAM_ROLE|ACCESS_KEY_ID"+ + "|SECRET_ACCESS_KEY|SESSION_TOKEN|MASTER_SYMMETRIC_KEY|KMS_KEY_ID"+ + "|EXTENSION|CLEANPATH|PARTITION|MAXFILESIZE\\s+\\d+\\s*(?:TB|PB))\\b");+private static final Pattern QUALIFIED_NAME = Pattern.compile("(?:[A-Za-z_][A-Za-z0-9_$]*|\"[^\"]+\")(?:\\.(?:[A-Za-z_][A-Za-z0-9_$]*|\"[^\"]+\"))?");private static final Pattern SIMPLE_NAME = Pattern.compile("[A-Za-z_][A-Za-z0-9_$]*|\"[^\"]+\"");@@ -61,11 +104,174 @@ public final class CopyStatementParser {private CopyStatementParser() {}- public static S3CopyFrom parse(String sql) {+ public static S3Statement parse(String sql) {if (sql == null || sql.isBlank()) {return null;}String cleaned = stripLeadingComments(sql.trim());+ if (cleaned.isEmpty()) {+ return null;+ }+ Matcher unload = UNLOAD_PATTERN.matcher(cleaned);+ if (unload.matches()) {+ return parseUnload(unload);+ }+ return parseCopy(cleaned);+ }++ private static S3Unload parseUnload(Matcher matcher) {+ String select = unescapeSingleQuotes(matcher.group(1).trim());+ if (!isSafeUnloadSubquery(select)) {+ return null;+ }+ String bucket = matcher.group(2);+ String prefix = matcher.group(3) != null ? matcher.group(3) : "";+ String options = matcher.group(4) != null ? matcher.group(4) : "";++ String flagScan = blankQuoted(options);+ if (UNLOAD_UNSUPPORTED_CLAUSE.matcher(flagScan).find()+ || TRAILING_STATEMENT.matcher(flagScan).find()) {+ return null;+ }+ int semi = flagScan.indexOf(';');+ if (semi >= 0) {+ if (!flagScan.substring(semi + 1).isBlank()) {+ return null;+ }+ options = options.substring(0, semi);+ }++ boolean csv = false;+ boolean gzip = false;+ boolean header = false;+ boolean addQuotes = false;+ boolean manifest = false;+ boolean allowOverwrite = false;+ boolean parallel = true;+ String nullAs = null;+ String delimiter = null;+ long maxFileSizeBytes = 0L;++ boolean seenCsv = false;+ boolean seenGzip = false;+ boolean seenHeader = false;+ boolean seenAddQuotes = false;+ boolean seenManifest = false;+ boolean seenAllowOverwrite = false;+ boolean seenParallel = false;+ boolean seenNull = false;+ boolean seenDelimiter = false;+ boolean seenMaxFileSize = false;++ Matcher csvM = CSV_PATTERN.matcher(options);+ Matcher gzipM = GZIP_PATTERN.matcher(options);+ Matcher headerM = HEADER_PATTERN.matcher(options);+ Matcher addQuotesM = ADDQUOTES_PATTERN.matcher(options);+ Matcher manifestM = MANIFEST_PATTERN.matcher(options);+ Matcher allowOverwriteM = ALLOWOVERWRITE_PATTERN.matcher(options);+ Matcher parallelM = PARALLEL_PATTERN.matcher(options);+ Matcher delimiterM = DELIMITER_PATTERN.matcher(options);+ Matcher nullM = NULL_AS_PATTERN.matcher(options);+ Matcher maxFileSizeM = MAXFILESIZE_PATTERN.matcher(options);++ int offset = 0;+ int len = options.length();+ while (offset < len) {+ while (offset < len && Character.isWhitespace(options.charAt(offset))) {+ offset++;+ }+ if (offset >= len) {+ break;+ }+ if (matchClause(maxFileSizeM, offset, len)) {+ if (seenMaxFileSize) {+ return null;+ }+ seenMaxFileSize = true;+ maxFileSizeBytes = maxFileSizeToBytes(maxFileSizeM.group(1), maxFileSizeM.group(2));+ if (maxFileSizeBytes > S3CopySimulator.UNLOAD_MAX_TOTAL_BYTES) {+ // A per-file size the simulator cannot buffer: fail open so PostgreSQL,+ // rather than an unrelated late "result too large" error, reports it.+ return null;+ }+ offset = maxFileSizeM.end();+ } else if (matchClause(csvM, offset, len)) {+ if (seenCsv) {+ return null;+ }+ seenCsv = true;+ csv = true;+ offset = csvM.end();+ } else if (matchClause(gzipM, offset, len)) {+ if (seenGzip) {+ return null;+ }+ seenGzip = true;+ gzip = true;+ offset = gzipM.end();+ } else if (matchClause(headerM, offset, len)) {+ if (seenHeader) {+ return null;+ }+ seenHeader = true;+ header = true;+ offset = headerM.end();+ } else if (matchClause(addQuotesM, offset, len)) {+ if (seenAddQuotes) {+ return null;+ }+ seenAddQuotes = true;+ addQuotes = true;+ offset = addQuotesM.end();+ } else if (matchClause(manifestM, offset, len)) {+ if (seenManifest) {+ return null;+ }+ seenManifest = true;+ manifest = true;+ offset = manifestM.end();+ } else if (matchClause(allowOverwriteM, offset, len)) {+ if (seenAllowOverwrite) {+ return null;+ }+ seenAllowOverwrite = true;+ allowOverwrite = true;+ offset = allowOverwriteM.end();+ } else if (matchClause(parallelM, offset, len)) {+ if (seenParallel) {+ return null;+ }+ seenParallel = true;+ String v = parallelM.group(1);+ parallel = v == null || v.equalsIgnoreCase("ON") || v.equalsIgnoreCase("TRUE");+ offset = parallelM.end();+ } else if (matchClause(delimiterM, offset, len)) {+ if (seenDelimiter) {+ return null;+ }+ seenDelimiter = true;+ delimiter = extractDelimiterValue(delimiterM);+ offset = delimiterM.end();+ } else if (matchClause(nullM, offset, len)) {+ if (seenNull) {+ return null;+ }+ seenNull = true;+ nullAs = extractNullValue(nullM);+ offset = nullM.end();+ } else {+ return null;+ }+ }++ if (delimiter == null) {+ delimiter = csv ? "," : "|";+ }+ return new S3Unload(select, bucket, prefix, delimiter, header, gzip, csv,+ addQuotes, nullAs, manifest, allowOverwrite, parallel, maxFileSizeBytes);+ }++ private static S3CopyFrom parseCopy(String cleaned) {Matcher matcher = COPY_PATTERN.matcher(cleaned);if (!matcher.matches()) {return null;@@ -272,4 +478,117 @@ public final class CopyStatementParser {}}}++ private static long maxFileSizeToBytes(String number, String unit) {+ double value = Double.parseDouble(number);+ long scale = 1L;+ if (unit != null && unit.equalsIgnoreCase("MB")) {+ scale = 1024L * 1024;+ } else if (unit != null && unit.equalsIgnoreCase("GB")) {+ scale = 1024L * 1024 * 1024;+ }+ long bytes = (long) (value * scale);+ return bytes > 0 ? bytes : 0L;+ }++ private static String unescapeSingleQuotes(String s) {+ return s.replace("''", "'");+ }++ /**+ * True only if the UNLOAD subquery cannot escape the {@code COPY (<q>) TO STDOUT}+ * wrapper it is spliced into: it must start with {@code SELECT} or {@code WITH},+ * keep parentheses balanced (never dipping below zero), and contain no {@code ;},+ * {@code $}, {@code --}, or {@code /*} outside a single- or double-quoted run.+ */+ private static boolean isSafeUnloadSubquery(String q) {+ if (!(startsWithKeyword(q, "SELECT") || startsWithKeyword(q, "WITH"))) {+ return false;+ }+ int depth = 0;+ boolean inSingle = false;+ boolean inDouble = false;+ for (int i = 0; i < q.length(); i++) {+ char c = q.charAt(i);+ if (inSingle) {+ if (c == '\'') {+ if (i + 1 < q.length() && q.charAt(i + 1) == '\'') {+ i++;+ } else {+ inSingle = false;+ }+ }+ continue;+ }+ if (inDouble) {+ if (c == '"') {+ inDouble = false;+ }+ continue;+ }+ switch (c) {+ case '\'' -> {+ if (i > 0) {+ char prev = q.charAt(i - 1);+ if (prev == 'E' || prev == 'e') {+ return false; // C-style escape string literal+ }+ if (prev == '&' && i > 1) {+ char prev2 = q.charAt(i - 2);+ if (prev2 == 'U' || prev2 == 'u') {+ return false; // Unicode escape string literal+ }+ }+ }+ inSingle = true;+ }+ case '"' -> {+ if (i > 1 && q.charAt(i - 1) == '&') {+ char prev2 = q.charAt(i - 2);+ if (prev2 == 'U' || prev2 == 'u') {+ return false; // Unicode escape identifier+ }+ }+ inDouble = true;+ }+ case '\\' -> {+ return false; // backslash outside quotes+ }+ case '(' -> depth++;+ case ')' -> {+ if (--depth < 0) {+ return false;+ }+ }+ case ';', '$' -> {+ return false;+ }+ case '-' -> {+ if (i + 1 < q.length() && q.charAt(i + 1) == '-') {+ return false;+ }+ }+ case '/' -> {+ if (i + 1 < q.length() && q.charAt(i + 1) == '*') {+ return false;+ }+ }+ default -> {+ }+ }+ }+ return depth == 0 && !inSingle && !inDouble;+ }++ private static boolean startsWithKeyword(String s, String keyword) {+ String t = s.stripLeading();+ if (!t.regionMatches(true, 0, keyword, 0, keyword.length())) {+ return false;+ }+ if (t.length() == keyword.length()) {+ return true;+ }+ char next = t.charAt(keyword.length());+ return !(Character.isLetterOrDigit(next) || next == '_' || next == '$');+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.javaindex 85d495e4db..db9c9bcf5c 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java@@ -102,18 +102,23 @@ public class RedshiftInterceptingBridge {String sql = msg.getSql();- CopyStatementParser.S3CopyFrom copyFrom = null;+ CopyStatementParser.S3Statement parsed = null;try {- copyFrom = CopyStatementParser.parse(sql);+ parsed = CopyStatementParser.parse(sql);} catch (RuntimeException e) {LOG.warnv("CopyStatementParser failed, forwarding original query: {0}", e.getMessage());}- if (copyFrom != null) {- CopyStatementParser.S3CopyFrom spec = copyFrom;- boolean intercepted = runWithBackendOwned(- () -> S3CopySimulator.runCopyFrom(client, backend, spec, s3Service, lastBackendStatus,- status -> lastBackendStatus = (char) status));+ if (parsed != null) {+ CopyStatementParser.S3Statement statement = parsed;+ boolean intercepted = runWithBackendOwned(() -> switch (statement) {+ case CopyStatementParser.S3CopyFrom c ->+ S3CopySimulator.runCopyFrom(client, backend, c, s3Service, lastBackendStatus,+ status -> lastBackendStatus = (char) status);+ case CopyStatementParser.S3Unload u ->+ S3CopySimulator.runUnload(client, backend, u, s3Service, lastBackendStatus,+ status -> lastBackendStatus = (char) status);+ });if (intercepted) {continue;}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.javaindex 3ff45c0f6d..a2145c3ccd 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java@@ -14,12 +14,17 @@ import java.net.Socket;import java.nio.charset.StandardCharsets;import java.util.ArrayList;import java.util.List;+import java.util.Map;+import java.util.concurrent.Semaphore;+import java.util.function.IntConsumer;import java.util.zip.GZIPInputStream;+import java.util.zip.GZIPOutputStream;/**- * Emulates {@code COPY <table> FROM 's3://bucket/keyOrPrefix'} by reading the S3 object (or every- * object under the prefix, in key order) through {@link S3Service} and streaming it into the- * backing PostgreSQL container over a fabricated {@code COPY <table> FROM STDIN} exchange.+ * Emulates {@code COPY <table> FROM 's3://bucket/keyOrPrefix'} and+ * {@code UNLOAD ('<select>') TO 's3://bucket/prefix'} by streaming data between {@link S3Service}+ * and the backing PostgreSQL container over a fabricated {@code COPY ... FROM STDIN} or+ * {@code COPY (...) TO STDOUT} exchange.** <p>The caller must own the backend socket exclusively for the whole call: this class writes a* query and reads its response frames directly. Every exit path leaves the client with exactly one@@ -33,6 +38,20 @@ public final class S3CopySimulator {private static final int LIST_PAGE_SIZE = 1000;private static final int CHUNK = 8192;+ // The two limits below are package-private and non-final only so unit tests can shrink them to+ // force multi-slice and over-limit paths on small inputs. Surefire runs a test class serially, and+ // each test restores the previous value in a finally block, so there is no cross-test interference.+ /** Default per-object size when the statement gives no MAXFILESIZE. Small because each slice is buffered in heap. */+ static long UNLOAD_TARGET_FILE_BYTES = 6L * 1024 * 1024;+ /** Whole-result ceiling; a larger UNLOAD is aborted rather than filling the in-memory S3 store. */+ static long UNLOAD_MAX_TOTAL_BYTES = 256L * 1024 * 1024;+ private static final long UNLOAD_WARN_BYTES = 32L * 1024 * 1024;+ /** Shared across every connection so concurrent UNLOADs cannot multiply the per-slice heap cost without bound. */+ static final Semaphore UNLOAD_HEAP_MIB = new Semaphore(192);+ private static final int UNLOAD_INITIAL_MIB = 12;++ private static final String SQLSTATE_PROGRAM_LIMIT_EXCEEDED = "54000";+ private static final String SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED = "53400";private static final String SQLSTATE_INSUFFICIENT_PRIVILEGE = "42501";private static final String SQLSTATE_INTERNAL = "XX000";@@ -54,7 +73,7 @@ public final class S3CopySimulator {public static boolean runCopyFrom(Socket client, Socket backend,CopyStatementParser.S3CopyFrom spec, S3Service s3,- char txStatus, java.util.function.IntConsumer onStatusChange) throws IOException {+ char txStatus, IntConsumer onStatusChange) throws IOException {try {s3.authorizeAnonymousListBucket(spec.bucket());} catch (AwsException e) {@@ -135,7 +154,7 @@ public final class S3CopySimulator {private static void abortOpenCopyIn(Socket client, Socket backend, OutputStream backendOut,PostgresWireDecoder backendDecoder, Exception cause,- char txStatus, java.util.function.IntConsumer onStatusChange) throws IOException {+ char txStatus, IntConsumer onStatusChange) throws IOException {try {writeCopyFail(backendOut, cause.getMessage());drainToReadyForQuery(backendDecoder, client, onStatusChange);@@ -276,7 +295,7 @@ public final class S3CopySimulator {}private static void drainToReadyForQuery(PostgresWireDecoder decoder, Socket client,- java.util.function.IntConsumer onStatusChange) throws IOException {+ IntConsumer onStatusChange) throws IOException {OutputStream clientOut = client.getOutputStream();while (true) {PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();@@ -300,13 +319,13 @@ public final class S3CopySimulator {}private static void sendAccessDenied(Socket client, Socket backend, CopyStatementParser.S3CopyFrom spec,- char txStatus, java.util.function.IntConsumer onStatusChange) throws IOException {+ char txStatus, IntConsumer onStatusChange) throws IOException {sendError(client, backend, SQLSTATE_INSUFFICIENT_PRIVILEGE,"S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), txStatus, onStatusChange);}private static void sendError(Socket client, Socket backend, String sqlState, String message,- char txStatus, java.util.function.IntConsumer onStatusChange) throws IOException {+ char txStatus, IntConsumer onStatusChange) throws IOException {if (txStatus == 'T' && backend != null && !backend.isClosed()) {if (!failBackendTransaction(backend, onStatusChange)) {closeQuietly(backend);@@ -340,7 +359,7 @@ public final class S3CopySimulator {}}- private static boolean failBackendTransaction(Socket backend, java.util.function.IntConsumer onStatusChange) {+ private static boolean failBackendTransaction(Socket backend, IntConsumer onStatusChange) {try {OutputStream out = backend.getOutputStream();out.write(PostgresWireDecoder.encodeQuery("(FLOCI_ABORT_TX)"));@@ -401,4 +420,459 @@ public final class S3CopySimulator {(byte) (value >>> 24), (byte) (value >>> 16), (byte) (value >>> 8), (byte) value};}++ public static boolean runUnload(Socket client, Socket backend,+ CopyStatementParser.S3Unload spec, S3Service s3, char txStatus) throws IOException {+ return runUnload(client, backend, spec, s3, txStatus, null);+ }++ public static boolean runUnload(Socket client, Socket backend,+ CopyStatementParser.S3Unload spec, S3Service s3, char txStatus,+ IntConsumer onStatusChange) throws IOException {++ String probeKey = unloadDataKey(spec, 0);+ String manifestKey = spec.prefix() + "manifest";+ try {+ s3.authorizeAnonymousPutObject(spec.bucket(), probeKey);+ if (spec.manifest()) {+ s3.authorizeAnonymousPutObject(spec.bucket(), manifestKey);+ }+ } catch (AwsException e) {+ LOG.debugv(e, "PutObject denied for UNLOAD to s3://{0}/{1}", spec.bucket(), spec.prefix());+ sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);+ return true;+ }++ try {+ if (!spec.allowOverwrite()) {+ s3.authorizeAnonymousListBucket(spec.bucket());+ if (targetPrefixHasObjects(spec, s3)) {+ sendError(client, backend, SQLSTATE_INTERNAL,+ "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()+ + " is not empty; specify ALLOWOVERWRITE to overwrite", txStatus, onStatusChange);+ return true;+ }+ }+ } catch (AwsException e) {+ LOG.debugv(e, "bucket check failed for UNLOAD to s3://{0}/{1}", spec.bucket(), spec.prefix());+ sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);+ return true;+ }++ if (!UNLOAD_HEAP_MIB.tryAcquire(UNLOAD_INITIAL_MIB)) {+ sendError(client, backend, SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,+ "UNLOAD memory budget exhausted; retry shortly", txStatus, onStatusChange);+ return true;+ }+ int[] heldMib = {UNLOAD_INITIAL_MIB};+ try {+ return runUnloadStreaming(client, backend, spec, s3, txStatus, onStatusChange, heldMib);+ } finally {+ UNLOAD_HEAP_MIB.release(heldMib[0]);+ }+ }++ private static boolean runUnloadStreaming(Socket client, Socket backend,+ CopyStatementParser.S3Unload spec, S3Service s3, char txStatus,+ IntConsumer onStatusChange, int[] heldMib) throws IOException {++ PostgresWireDecoder.FrontendMessage cmdComplete = null;+ PostgresWireDecoder.FrontendMessage readyForQuery = null;++ OutputStream backendOut = backend.getOutputStream();+ backendOut.write(PostgresWireDecoder.encodeQuery(fabricateUnloadCopy(spec)));+ backendOut.flush();++ PostgresWireDecoder backendDecoder = new PostgresWireDecoder(backend.getInputStream());+ PostgresWireDecoder.FrontendMessage first;+ try {+ first = nextNonAsync(backendDecoder, client);+ } catch (IOException e) {+ LOG.warnv(e, "backend read failed while awaiting CopyOutResponse");+ closeQuietly(backend);+ sendError(client, null, SQLSTATE_INTERNAL,+ "UNLOAD failed: backend closed or timed out", txStatus, onStatusChange);+ closeQuietly(client);+ return true;+ }+ if (first == null) {+ closeQuietly(backend);+ sendError(client, null, SQLSTATE_INTERNAL,+ "UNLOAD failed: backend closed before COPY started", txStatus, onStatusChange);+ closeQuietly(client);+ return true;+ }+ if (first.type() != 'H') {+ // Backend rejected the SELECT itself. Its ErrorResponse + ReadyForQuery are the client's one response.+ forward(client, first);+ drainToReadyForQuery(backendDecoder, client, onStatusChange);+ return true;+ }++ long threshold = spec.maxFileSizeBytes() > 0 ? spec.maxFileSizeBytes() : UNLOAD_TARGET_FILE_BYTES;+ String contentType = spec.gzip() ? "application/gzip" : "text/plain";+ List<String> writtenKeys = new ArrayList<>();+ List<Integer> writtenLengths = new ArrayList<>();++ ByteArrayOutputStream sink = new ByteArrayOutputStream();+ OutputStream acc = spec.gzip() ? new GZIPOutputStream(sink) : sink;+ long rawSlice = 0;+ // Data-row bytes in the current slice, excluding a repeated header. Drives the size split so a+ // header-only slice is never cut off on its own.+ long slicePayload = 0;+ long rawTotal = 0;+ boolean lastByteNewline = true;+ boolean warned = false;+ int sliceIndex = 0;++ // With HEADER the backend emits the header row once, at the top of the stream. Capture it so+ // it can be repeated at the start of every later slice, the way real Redshift writes one+ // header per output file.+ ByteArrayOutputStream headerBuf = spec.header() ? new ByteArrayOutputStream() : null;+ byte[] headerBytes = null;+ boolean capturingHeader = spec.header();++ try {+ while (true) {+ PostgresWireDecoder.FrontendMessage m = backendDecoder.nextMessage();+ if (m == null) {+ closeAcc(acc, sink);+ closeQuietly(backend);+ if (spec.manifest()) {+ deleteWritten(s3, spec, writtenKeys);+ }+ sendError(client, null, SQLSTATE_INTERNAL,+ "UNLOAD failed: backend closed mid-stream", txStatus, onStatusChange);+ closeQuietly(client);+ return true;+ }+ char t = m.type();+ if (t == 'd') {+ byte[] body = m.body();+ int dataStart = 0;+ if (capturingHeader) {+ int nl = -1;+ for (int i = 0; i < body.length; i++) {+ if (body[i] == '\n') {+ nl = i;+ break;+ }+ }+ int end = nl >= 0 ? nl + 1 : body.length;+ headerBuf.write(body, 0, end);+ if (nl >= 0) {+ headerBytes = headerBuf.toByteArray();+ headerBuf = null;+ capturingHeader = false;+ }+ dataStart = end;+ } else if (headerBytes != null && sliceIndex > 0 && rawSlice == 0) {+ // First data of a later slice: repeat the captured header row.+ acc.write(headerBytes);+ rawSlice += headerBytes.length;+ }+ acc.write(body, 0, body.length);+ rawSlice += body.length;+ rawTotal += body.length;+ slicePayload += body.length - dataStart;+ if (body.length > 0) {+ lastByteNewline = body[body.length - 1] == '\n';+ }+ if (!warned && rawTotal > UNLOAD_WARN_BYTES) {+ warned = true;+ LOG.warnv("UNLOAD result passed {0} bytes and is buffered a slice at a time "+ + "(bucket={1}, prefix={2})", UNLOAD_WARN_BYTES, spec.bucket(), spec.prefix());+ }+ int wantMib = (int) ((rawSlice * 3) / (1024 * 1024)) + 1;+ while (heldMib[0] < wantMib) {+ if (!UNLOAD_HEAP_MIB.tryAcquire(1)) {+ return abortUnload(client, backend, backendDecoder, acc, sink,+ SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,+ "UNLOAD memory budget exhausted; retry with a smaller result",+ txStatus, onStatusChange);+ }+ heldMib[0]++;+ }+ if (rawTotal > UNLOAD_MAX_TOTAL_BYTES) {+ closeAcc(acc, sink);+ closeQuietly(backend);+ if (spec.manifest()) {+ deleteWritten(s3, spec, writtenKeys);+ }+ sendError(client, null,+ SQLSTATE_PROGRAM_LIMIT_EXCEEDED,+ "UNLOAD result exceeds the " + UNLOAD_MAX_TOTAL_BYTES + "-byte limit",+ txStatus, onStatusChange);+ closeQuietly(client);+ return true;+ }+ if (slicePayload >= threshold && lastByteNewline) {+ byte[] payload = finishSlice(acc, sink);+ String key = unloadDataKey(spec, sliceIndex);+ try {+ s3.authorizeAnonymousPutObject(spec.bucket(), key);+ s3.putObject(spec.bucket(), key, payload, contentType, Map.of());+ } catch (AwsException e) {+ LOG.debugv(e, "UNLOAD slice write to s3://{0}/{1} failed", spec.bucket(), key);+ if (spec.manifest()) {+ deleteWritten(s3, spec, writtenKeys);+ }+ return abortUnload(client, backend, backendDecoder, sink, sink,+ unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);+ }+ writtenKeys.add(key);+ writtenLengths.add(payload.length);+ sliceIndex++;+ int release = heldMib[0] - UNLOAD_INITIAL_MIB;+ if (release > 0) {+ UNLOAD_HEAP_MIB.release(release);+ heldMib[0] = UNLOAD_INITIAL_MIB;+ }+ rawSlice = 0;+ slicePayload = 0;+ lastByteNewline = true;+ sink = new ByteArrayOutputStream();+ acc = spec.gzip() ? new GZIPOutputStream(sink) : sink;+ }+ } else if (t == 'c') {+ // CopyDone, ignore+ } else if (t == 'C') {+ // CommandComplete, hold until we see 'Z'+ cmdComplete = m;+ } else if (t == 'Z') {+ readyForQuery = m;+ break;+ } else if (t == 'E') {+ closeAcc(acc, sink);+ if (spec.manifest()) {+ deleteWritten(s3, spec, writtenKeys);+ }+ forward(client, m);+ drainToReadyForQuery(backendDecoder, client, onStatusChange);+ return true;+ } else if (t == 'N' || t == 'A' || t == 'S') {+ forward(client, m);+ }+ }+ } catch (AwsException e) {+ LOG.debugv(e, "UNLOAD S3 operation failed during streaming");+ if (spec.manifest()) {+ deleteWritten(s3, spec, writtenKeys);+ }+ return abortUnload(client, backend, backendDecoder, acc, sink,+ unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);+ } catch (RuntimeException | IOException e) {+ LOG.warnv(e, "UNLOAD streaming failed");+ if (spec.manifest()) {+ deleteWritten(s3, spec, writtenKeys);+ }+ String detail = e.getMessage() != null ? e.getMessage() : e.toString();+ return abortUnload(client, backend, backendDecoder, acc, sink,+ SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, txStatus, onStatusChange);+ }++ // Flush the final slice. Write it when it carries rows, or when nothing has been written yet+ // (a zero-row UNLOAD still emits one object, header only when HEADER was asked for). A result+ // that ended exactly on a slice boundary leaves slicePayload == 0 with slices already written,+ // so neither an empty nor a header-only trailer is produced.+ byte[] payload = finishSlice(acc, sink);+ if (slicePayload > 0 || writtenKeys.isEmpty()) {+ String key = unloadDataKey(spec, sliceIndex);+ try {+ s3.authorizeAnonymousPutObject(spec.bucket(), key);+ s3.putObject(spec.bucket(), key, payload, contentType, Map.of());+ } catch (AwsException e) {+ LOG.debugv(e, "UNLOAD final slice write to s3://{0}/{1} failed", spec.bucket(), key);+ if (spec.manifest()) {+ deleteWritten(s3, spec, writtenKeys);+ }+ sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec),+ txStatus, onStatusChange);+ return true;+ } catch (RuntimeException e) {+ LOG.warnv(e, "UNLOAD final slice write failed");+ if (spec.manifest()) {+ deleteWritten(s3, spec, writtenKeys);+ }+ String detail = e.getMessage() != null ? e.getMessage() : e.toString();+ sendError(client, backend, SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, txStatus, onStatusChange);+ return true;+ }+ writtenKeys.add(key);+ writtenLengths.add(payload.length);+ }++ if (spec.manifest()) {+ try {+ s3.authorizeAnonymousPutObject(spec.bucket(), spec.prefix() + "manifest");+ s3.putObject(spec.bucket(), spec.prefix() + "manifest",+ manifestJson(spec.bucket(), writtenKeys, writtenLengths).getBytes(StandardCharsets.UTF_8),+ "application/json", Map.of());+ } catch (RuntimeException e) {+ LOG.warnv(e, "UNLOAD manifest write failed; removing partial data objects");+ deleteWritten(s3, spec, writtenKeys);+ sendError(client, backend, SQLSTATE_INTERNAL, "UNLOAD manifest write failed", txStatus, onStatusChange);+ return true;+ }+ }++ if (cmdComplete != null) {+ forward(client, cmdComplete);+ }+ forward(client, readyForQuery);+ if (onStatusChange != null && readyForQuery.body().length > 0) {+ onStatusChange.accept(readyForQuery.body()[0]);+ }+ return true;+ }++ private static boolean targetPrefixHasObjects(CopyStatementParser.S3Unload spec, S3Service s3) {+ S3Service.ListObjectsResult r = s3.listObjectsWithPrefixes(+ spec.bucket(), spec.prefix(), null, 1, null, null);+ return r != null && r.objects() != null && !r.objects().isEmpty();+ }++ private static String fabricateUnloadCopy(CopyStatementParser.S3Unload spec) {+ boolean csvFraming = spec.csv() || spec.addQuotes() || spec.header();+ StringBuilder sql = new StringBuilder("COPY (").append(spec.selectQuery())+ .append(") TO STDOUT WITH (FORMAT ").append(csvFraming ? "csv" : "text");+ String delimiter = spec.delimiter() != null ? spec.delimiter() : (csvFraming ? "," : "|");+ sql.append(", DELIMITER '").append(quoteLiteral(delimiter)).append("'");+ if (spec.header()) {+ sql.append(", HEADER true");+ }+ if (spec.addQuotes()) {+ sql.append(", FORCE_QUOTE *");+ }+ if (spec.nullAs() != null) {+ sql.append(", NULL '").append(quoteLiteral(spec.nullAs())).append("'");+ }+ sql.append(")");+ return sql.toString();+ }++ /** {@code 404} from the S3 layer means the bucket is absent, not that access was denied. */+ private static String unloadWriteSqlState(AwsException e) {+ return e.getHttpStatus() == 404 ? SQLSTATE_INTERNAL : SQLSTATE_INSUFFICIENT_PRIVILEGE;+ }++ private static String unloadWriteMessage(AwsException e, CopyStatementParser.S3Unload spec) {+ return e.getHttpStatus() == 404+ ? "S3 bucket s3://" + spec.bucket() + " not found"+ : "S3 access denied for s3://" + spec.bucket() + "/" + spec.prefix();+ }++ private static String unloadDataKey(CopyStatementParser.S3Unload spec, int index) {+ String base = spec.parallel()+ ? spec.prefix() + String.format("%04d_part_00", index)+ : spec.prefix() + String.format("%03d", index);+ return spec.gzip() ? base + ".gz" : base;+ }++ private static byte[] finishSlice(OutputStream acc, ByteArrayOutputStream sink) throws IOException {+ if (acc != sink) {+ acc.close(); // flush the GZIP trailer+ }+ return sink.toByteArray();+ }++ private static void closeAcc(OutputStream acc, ByteArrayOutputStream sink) {+ if (acc != sink) {+ try {+ acc.close();+ } catch (IOException e) {+ LOG.debugv(e, "error closing the UNLOAD accumulation stream");+ }+ }+ }++ private static boolean abortUnload(Socket client, Socket backend, PostgresWireDecoder backendDecoder,+ OutputStream acc, ByteArrayOutputStream sink, String sqlState, String message,+ char txStatus, IntConsumer onStatusChange) throws IOException {+ closeAcc(acc, sink);+ try {+ drainBackendDiscarding(backendDecoder);+ } catch (IOException backendGone) {+ // The backend is unreachable, so it cannot be resynchronized: close it and hand the+ // client its one response synthesized, exactly as the backend-EOF branch does.+ LOG.debugv(backendGone, "backend unreachable while draining an aborted UNLOAD; synthesizing client error");+ closeQuietly(backend);+ sendError(client, null, sqlState, message, txStatus, onStatusChange);+ closeQuietly(client);+ return true;+ }+ sendError(client, backend, sqlState, message, txStatus, onStatusChange);+ return true;+ }++ /** Read backend messages, forwarding nothing, until its ReadyForQuery or EOF. Leaves the backend socket clean. */+ private static void drainBackendDiscarding(PostgresWireDecoder decoder) throws IOException {+ PostgresWireDecoder.FrontendMessage m;+ while ((m = decoder.nextMessage()) != null) {+ if (m.type() == 'Z') {+ return;+ }+ }+ }++ private static void deleteWritten(S3Service s3, CopyStatementParser.S3Unload spec, List<String> keys) {+ if (spec.allowOverwrite()) {+ // Under ALLOWOVERWRITE a slice key may have replaced a pre-existing object, and there is no+ // reliable way to tell an overwrite from a fresh write (the check and the write are not+ // atomic, and a concurrent UNLOAD to the same prefix could own the stored object). Rather+ // than risk deleting data this operation did not create, leave every written object in+ // place. A failed MANIFEST + ALLOWOVERWRITE UNLOAD can therefore leave valid data objects+ // without a manifest; rerunning the same statement overwrites them cleanly.+ return;+ }+ // Non-overwrite runs verified the prefix was empty before streaming, so every written key was+ // created by this operation and is safe to remove.+ for (String k : keys) {+ try {+ s3.authorizeAnonymousDeleteObject(spec.bucket(), k);+ s3.deleteObject(spec.bucket(), k);+ } catch (RuntimeException e) {+ LOG.debugv(e, "could not remove partial UNLOAD object s3://{0}/{1}", spec.bucket(), k);+ }+ }+ }++ private static String manifestJson(String bucket, List<String> keys, List<Integer> lengths) {+ StringBuilder json = new StringBuilder("{\"entries\":[");+ for (int i = 0; i < keys.size(); i++) {+ if (i > 0) {+ json.append(',');+ }+ json.append("{\"url\":\"s3://").append(escapeJson(bucket)).append('/').append(escapeJson(keys.get(i)))+ .append("\",\"meta\":{\"content_length\":").append(lengths.get(i)).append("}}");+ }+ return json.append("]}").toString();+ }++ private static String escapeJson(String s) {+ if (s == null) {+ return "";+ }+ StringBuilder sb = new StringBuilder(s.length() + 8);+ for (int i = 0; i < s.length(); i++) {+ char c = s.charAt(i);+ switch (c) {+ case '"' -> sb.append("\\\"");+ case '\\' -> sb.append("\\\\");+ case '\b' -> sb.append("\\b");+ case '\f' -> sb.append("\\f");+ case '\n' -> sb.append("\\n");+ case '\r' -> sb.append("\\r");+ case '\t' -> sb.append("\\t");+ default -> {+ if (c < ' ') {+ sb.append(String.format("\\u%04x", (int) c));+ } else {+ sb.append(c);+ }+ }+ }+ }+ return sb.toString();+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex de6a63d597..490e919417 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -617,6 +617,16 @@ public class S3Service implements Resettable, ResourceProvider {authorizeListBucket(bucketName, RequestAuthorization.unsigned());}+ /** Authorize an unsigned {@code s3:PutObject}; see {@link #authorizeAnonymousGetObject}. */+ public void authorizeAnonymousPutObject(String bucketName, String key) {+ authorizePutObject(bucketName, key, RequestAuthorization.unsigned());+ }++ /** Authorize an unsigned {@code s3:DeleteObject}; see {@link #authorizeAnonymousGetObject}. */+ public void authorizeAnonymousDeleteObject(String bucketName, String key) {+ authorizeDeleteObject(bucketName, key, null, RequestAuthorization.unsigned());+ }+public void authorizeCloudFrontOacGetObject(String bucketName, String key, String distributionArn) {authorizeCloudFrontGetObject(diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.javaindex 9b12d1ff44..9fe447dd54 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java@@ -1,6 +1,8 @@package io.github.hectorvent.floci.services.redshift;import io.github.hectorvent.floci.services.redshift.model.Cluster;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.s3.model.S3Object;import io.quarkus.test.junit.QuarkusTest;import jakarta.inject.Inject;import org.awaitility.Awaitility;@@ -10,15 +12,25 @@ import org.junit.jupiter.api.Assumptions;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.Test;+import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;+import java.nio.charset.StandardCharsets;import java.sql.Connection;import java.sql.DriverManager;import java.sql.ResultSet;import java.sql.SQLException;import java.sql.Statement;import java.time.Duration;+import java.util.Comparator;+import java.util.List;+import java.util.Map;import java.util.Objects;+import java.util.zip.GZIPInputStream;+import java.util.zip.GZIPOutputStream;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTest@@ -28,7 +40,7 @@ class RedshiftInterceptorIntegrationTest {RedshiftService service;@Inject- io.github.hectorvent.floci.services.s3.S3Service s3;+ S3Service s3;private String clusterId;@@ -118,7 +130,7 @@ class RedshiftInterceptorIntegrationTest {String bucket = "redshift-copy-it";s3.createBucket(bucket, "us-east-1");s3.putObject(bucket, "people/p1.txt",- "1|alice\n2|bob\n".getBytes(java.nio.charset.StandardCharsets.UTF_8), "text/plain", java.util.Map.of());+ "1|alice\n2|bob\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {c.createStatement().execute("CREATE TABLE people (id int, name text)");@@ -137,8 +149,8 @@ class RedshiftInterceptorIntegrationTest {String bucket = "redshift-copy-it-prefix";s3.createBucket(bucket, "us-east-1");- s3.putObject(bucket, "d/a", "1|a\n".getBytes(java.nio.charset.StandardCharsets.UTF_8), "text/plain", java.util.Map.of());- s3.putObject(bucket, "d/b", "2|b\n".getBytes(java.nio.charset.StandardCharsets.UTF_8), "text/plain", java.util.Map.of());+ s3.putObject(bucket, "d/a", "1|a\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());+ s3.putObject(bucket, "d/b", "2|b\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {c.createStatement().execute("CREATE TABLE t (id int, v text)");@@ -157,11 +169,11 @@ class RedshiftInterceptorIntegrationTest {String bucket = "redshift-copy-it-gzip";s3.createBucket(bucket, "us-east-1");- java.io.ByteArrayOutputStream raw = new java.io.ByteArrayOutputStream();- try (java.util.zip.GZIPOutputStream gz = new java.util.zip.GZIPOutputStream(raw)) {- gz.write("10|x\n11|y\n".getBytes(java.nio.charset.StandardCharsets.UTF_8));+ ByteArrayOutputStream raw = new ByteArrayOutputStream();+ try (GZIPOutputStream gz = new GZIPOutputStream(raw)) {+ gz.write("10|x\n11|y\n".getBytes(StandardCharsets.UTF_8));}- s3.putObject(bucket, "g/data.gz", raw.toByteArray(), "application/gzip", java.util.Map.of());+ s3.putObject(bucket, "g/data.gz", raw.toByteArray(), "application/gzip", Map.of());try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {c.createStatement().execute("CREATE TABLE g (id int, v text)");@@ -183,7 +195,7 @@ class RedshiftInterceptorIntegrationTest {try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {c.createStatement().execute("CREATE TABLE t2 (id int)");- SQLException ex = org.junit.jupiter.api.Assertions.assertThrows(+ SQLException ex = assertThrows(SQLException.class,() -> c.createStatement().execute("COPY t2 FROM 's3://redshift-copy-it-missing/does/not/exist'"));assertTrue(@@ -204,12 +216,12 @@ class RedshiftInterceptorIntegrationTest {c.setAutoCommit(false);Statement st = c.createStatement();st.execute("INSERT INTO tx_test VALUES (1)");- org.junit.jupiter.api.Assertions.assertThrows(+ assertThrows(SQLException.class,() -> st.execute("COPY tx_test FROM 's3://redshift-copy-it-tx/does/not/exist'"));// The backend must now be in the aborted transaction state ('E').// Subsequent statements in this transaction block must fail.- org.junit.jupiter.api.Assertions.assertThrows(+ assertThrows(SQLException.class,() -> st.execute("INSERT INTO tx_test VALUES (2)"));c.rollback();@@ -220,4 +232,99 @@ class RedshiftInterceptorIntegrationTest {}}}++ @Test+ void unloadToS3PrefixWritesRowsAsObjects() throws Exception {+ clusterId = "it-unload-basic";+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");++ String bucket = "redshift-unload-it";+ s3.createBucket(bucket, "us-east-1");++ try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {+ c.createStatement().execute("CREATE TABLE u_src (id int, name text)");+ c.createStatement().execute("INSERT INTO u_src VALUES (1, 'alice'), (2, 'bob')");+ c.createStatement().execute(+ "UNLOAD ('select id, name from u_src order by id') TO 's3://redshift-unload-it/out/'");+ }++ List<S3Object> objs =+ s3.listObjects(bucket, "out/", null, 100);+ assertTrue(objs.size() >= 1, "UNLOAD must write at least one object");+ StringBuilder all = new StringBuilder();+ objs.stream()+ .sorted(Comparator.comparing(S3Object::getKey))+ // listObjects returns metadata-only entries; fetch each body with getObject.+ .forEach(o -> all.append(new String(+ s3.getObject(bucket, o.getKey()).getData(), StandardCharsets.UTF_8)));+ assertEquals("1|alice\n2|bob\n", all.toString());+ }++ @Test+ void unloadGzipWritesCompressedObject() throws Exception {+ clusterId = "it-unload-gzip";+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");+ String bucket = "redshift-unload-it-gzip";+ s3.createBucket(bucket, "us-east-1");++ try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {+ c.createStatement().execute("CREATE TABLE g_src (id int)");+ c.createStatement().execute("INSERT INTO g_src VALUES (7), (8)");+ c.createStatement().execute("UNLOAD ('select id from g_src order by id') TO 's3://redshift-unload-it-gzip/g/' GZIP");+ }++ var objs = s3.listObjects(bucket, "g/", null, 100);+ assertEquals(1, objs.size());+ assertTrue(objs.get(0).getKey().endsWith(".gz"), objs.get(0).getKey());+ byte[] data = s3.getObject(bucket, objs.get(0).getKey()).getData();+ ByteArrayOutputStream raw = new ByteArrayOutputStream();+ try (var in = new GZIPInputStream(new ByteArrayInputStream(data))) {+ in.transferTo(raw);+ }+ assertEquals("7\n8\n", raw.toString(StandardCharsets.UTF_8));+ }++ @Test+ void unloadWithManifestWritesAManifestObject() throws Exception {+ clusterId = "it-unload-manifest";+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");+ String bucket = "redshift-unload-it-manifest";+ s3.createBucket(bucket, "us-east-1");++ try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {+ c.createStatement().execute("CREATE TABLE m_src (id int)");+ c.createStatement().execute("INSERT INTO m_src VALUES (1)");+ c.createStatement().execute("UNLOAD ('select id from m_src') TO 's3://redshift-unload-it-manifest/m/' MANIFEST");+ }++ var manifest = s3.getObject(bucket, "m/manifest");+ assertNotNull(manifest);+ String json = new String(manifest.getData(), StandardCharsets.UTF_8);+ assertTrue(json.contains("\"entries\""), json);+ assertTrue(json.contains("s3://redshift-unload-it-manifest/m/"), json);+ }++ @Test+ void unloadIntoNonEmptyPrefixWithoutAllowOverwriteRaisesSqlError() throws Exception {+ clusterId = "it-unload-overwrite";+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");+ String bucket = "redshift-unload-it-ow";+ s3.createBucket(bucket, "us-east-1");+ s3.putObject(bucket, "o/existing", "x".getBytes(StandardCharsets.UTF_8),+ "text/plain", Map.of());++ try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {+ c.createStatement().execute("CREATE TABLE o_src (id int)");+ c.createStatement().execute("INSERT INTO o_src VALUES (1)");+ SQLException ex = assertThrows(SQLException.class,+ () -> c.createStatement().execute("UNLOAD ('select id from o_src') TO 's3://redshift-unload-it-ow/o/'"));+ assertTrue(ex.getMessage().toLowerCase().contains("allowoverwrite"), ex.getMessage());+ }++ // With ALLOWOVERWRITE the same statement succeeds.+ try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {+ c.createStatement().execute(+ "UNLOAD ('select id from o_src') TO 's3://redshift-unload-it-ow/o/' ALLOWOVERWRITE");+ }+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.javaindex c47a26f99b..a7ecb616ac 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftOperationsTest.java@@ -17,6 +17,7 @@ import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.containsString;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.when;@QuarkusTest@@ -110,7 +111,7 @@ public class RedshiftOperationsTest {void testClusterAndSnapshotLifecycle() {when(containerManager.start(any(), eq("cluster-src"), any(), any())).thenReturn(new RedshiftContainerHandle("c1", "cluster-src", "localhost", 5439));- org.mockito.Mockito.doAnswer(invocation -> {+ doAnswer(invocation -> {Path p = invocation.getArgument(3);Files.writeString(p, "-- dump sql table test_data;");return null;diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.javaindex 06807ed2b0..1fb0e2bd0f 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftQueryHandlerTest.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.redshift;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.redshift.model.Cluster;import io.github.hectorvent.floci.services.redshift.model.ClusterParameterGroup;import io.github.hectorvent.floci.services.redshift.model.ClusterSubnetGroup;@@ -17,6 +18,7 @@ import java.util.Map;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.Mockito.*;@@ -334,8 +336,8 @@ class RedshiftQueryHandlerTest {MultivaluedMap<String, String> params = new MultivaluedHashMap<>();params.putSingle("Description", "desc");- io.github.hectorvent.floci.core.common.AwsException ex = org.junit.jupiter.api.Assertions.assertThrows(- io.github.hectorvent.floci.core.common.AwsException.class,+ AwsException ex = assertThrows(+ AwsException.class,() -> handler.handle("CreateClusterSubnetGroup", params));assertEquals("InvalidParameterValue", ex.getErrorCode());}@@ -345,8 +347,8 @@ class RedshiftQueryHandlerTest {MultivaluedMap<String, String> params = new MultivaluedHashMap<>();params.putSingle("NodeType", "dc2.large");- io.github.hectorvent.floci.core.common.AwsException ex = org.junit.jupiter.api.Assertions.assertThrows(- io.github.hectorvent.floci.core.common.AwsException.class,+ AwsException ex = assertThrows(+ AwsException.class,() -> handler.handle("ModifyCluster", params));assertEquals("InvalidParameterValue", ex.getErrorCode());}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.javaindex 64850dc194..0c2cebe1e3 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java@@ -1,6 +1,9 @@package io.github.hectorvent.floci.services.redshift;+import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.redshift.container.RedshiftContainerHandle;@@ -11,6 +14,7 @@ import io.github.hectorvent.floci.services.redshift.model.ClusterSubnetGroup;import io.github.hectorvent.floci.services.redshift.model.Endpoint;import io.github.hectorvent.floci.services.redshift.model.Parameter;import io.github.hectorvent.floci.services.redshift.model.Snapshot;+import io.github.hectorvent.floci.services.redshift.proxy.RedshiftProxyManager;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.mockito.ArgumentCaptor;@@ -39,9 +43,9 @@ class RedshiftServiceTest {private AccountAwareStorageBackend<ClusterParameterGroup> parameterGroupBackend;private AccountAwareStorageBackend<ClusterSubnetGroup> subnetGroupBackend;private RedshiftContainerManager cm;- private io.github.hectorvent.floci.core.common.RegionResolver regionResolver;- private io.github.hectorvent.floci.services.redshift.proxy.RedshiftProxyManager proxyManager;- private io.github.hectorvent.floci.core.common.docker.DockerHostResolver dockerHostResolver;+ private RegionResolver regionResolver;+ private RedshiftProxyManager proxyManager;+ private DockerHostResolver dockerHostResolver;private RedshiftService service;@BeforeEach@@ -54,19 +58,19 @@ class RedshiftServiceTest {parameterGroupBackend = mock(AccountAwareStorageBackend.class);subnetGroupBackend = mock(AccountAwareStorageBackend.class);cm = mock(RedshiftContainerManager.class);- proxyManager = mock(io.github.hectorvent.floci.services.redshift.proxy.RedshiftProxyManager.class);- dockerHostResolver = mock(io.github.hectorvent.floci.core.common.docker.DockerHostResolver.class);+ proxyManager = mock(RedshiftProxyManager.class);+ dockerHostResolver = mock(DockerHostResolver.class);when(dockerHostResolver.resolve()).thenReturn("localhost");- io.github.hectorvent.floci.config.EmulatorConfig config = mock(io.github.hectorvent.floci.config.EmulatorConfig.class);- io.github.hectorvent.floci.config.EmulatorConfig.StorageConfig storageConfig = mock(io.github.hectorvent.floci.config.EmulatorConfig.StorageConfig.class);+ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.StorageConfig storageConfig = mock(EmulatorConfig.StorageConfig.class);when(config.storage()).thenReturn(storageConfig);when(storageConfig.persistentPath()).thenReturn("target/test-data");- io.github.hectorvent.floci.config.EmulatorConfig.ServicesConfig servicesConfig =- mock(io.github.hectorvent.floci.config.EmulatorConfig.ServicesConfig.class);- io.github.hectorvent.floci.config.EmulatorConfig.RedshiftServiceConfig redshiftConfig =- mock(io.github.hectorvent.floci.config.EmulatorConfig.RedshiftServiceConfig.class);+ EmulatorConfig.ServicesConfig servicesConfig =+ mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.RedshiftServiceConfig redshiftConfig =+ mock(EmulatorConfig.RedshiftServiceConfig.class);when(config.services()).thenReturn(servicesConfig);when(servicesConfig.redshift()).thenReturn(redshiftConfig);when(redshiftConfig.proxyBasePort()).thenReturn(7100);@@ -79,7 +83,7 @@ class RedshiftServiceTest {when(sf.<ClusterSubnetGroup>create(eq("redshift"), eq("redshift-subnet-groups.json"), any())).thenReturn(subnetGroupBackend);when(clusterBackend.accountId()).thenReturn("111111111111");- regionResolver = new io.github.hectorvent.floci.core.common.RegionResolver("us-east-1", "111111111111");+ regionResolver = new RegionResolver("us-east-1", "111111111111");service = new RedshiftService(sf, cm, config, regionResolver, proxyManager, dockerHostResolver);}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.javaindex e6a8cb3c72..a8d02b3020 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/CopyStatementParserTest.java@@ -6,14 +6,21 @@ import java.util.List;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;class CopyStatementParserTest {+ /** parse() returns S3Statement now; every COPY test wants the S3CopyFrom view. */+ private static CopyStatementParser.S3CopyFrom copyFrom(String sql) {+ CopyStatementParser.S3Statement s = CopyStatementParser.parse(sql);+ return (CopyStatementParser.S3CopyFrom) s;+ }+@Testvoid parsesMinimalCopyFromKey() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY sales FROM 's3://warehouse/data/sales.txt'");assertEquals("sales", c.targetTable());assertEquals(List.of(), c.columns());@@ -28,7 +35,7 @@ class CopyStatementParserTest {@Testvoid parsesColumnsOptionsAndPrefix() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY public.events (id, ts, note) FROM 's3://bkt/evt/' "+ "GZIP DELIMITER ',' IGNOREHEADER 2 NULL AS '\\\\N' FORMAT AS CSV");assertEquals("public.events", c.targetTable());@@ -44,7 +51,7 @@ class CopyStatementParserTest {@Testvoid defaultsDelimiterToCommaForCsvAndTreatsHeaderKeywordAsOneLine() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://b/k' CSV HEADER");assertEquals(",", c.delimiter());assertEquals(1, c.headerLines());@@ -52,21 +59,21 @@ class CopyStatementParserTest {@Testvoid decodesTabDelimiterToken() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://b/k' DELIMITER '\\\\t'");assertEquals("\t", c.delimiter());}@Testvoid bucketOnlyPathGivesEmptyPrefix() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse("COPY t FROM 's3://only-bucket'");+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://only-bucket'");assertEquals("only-bucket", c.bucket());assertEquals("", c.keyOrPrefix());}@Testvoid stripsLeadingComments() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(+ CopyStatementParser.S3CopyFrom c = copyFrom("-- load nightly\n/* batch */ COPY t FROM 's3://b/k'");assertEquals("t", c.targetTable());}@@ -95,7 +102,7 @@ class CopyStatementParserTest {@Testvoid quotedIdentifiersSurvive() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY \"My Schema\".\"Tab\" (\"col one\") FROM 's3://b/k'");assertEquals("\"My Schema\".\"Tab\"", c.targetTable());assertEquals(List.of("\"col one\""), c.columns());@@ -103,7 +110,7 @@ class CopyStatementParserTest {@Testvoid explicitDelimiterOverridesCsvDefault() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://b/k' CSV DELIMITER '\t'");assertEquals("\t", c.delimiter());assertTrue(c.csv());@@ -127,13 +134,13 @@ class CopyStatementParserTest {@Testvoid toleratesALoneTrailingSemicolon() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse("COPY t FROM 's3://b/k' GZIP;");+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://b/k' GZIP;");assertTrue(c.gzip());}@Testvoid aQuotedKeywordInAnOptionValueDoesNotChangeParsing() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://b/k' NULL AS 'csv' DELIMITER ';'");assertFalse(c.csv());assertEquals(";", c.delimiter());@@ -164,8 +171,116 @@ class CopyStatementParserTest {@Testvoid parsesEscapedSingleQuoteInDelimiter() {- CopyStatementParser.S3CopyFrom c = CopyStatementParser.parse(+ CopyStatementParser.S3CopyFrom c = copyFrom("COPY t FROM 's3://b/k' DELIMITER ''''");assertEquals("'", c.delimiter());}++ private static CopyStatementParser.S3Unload unload(String sql) {+ return (CopyStatementParser.S3Unload) CopyStatementParser.parse(sql);+ }++ @Test+ void parsesMinimalUnload() {+ CopyStatementParser.S3Unload u = unload(+ "UNLOAD ('select id, name from sales') TO 's3://warehouse/out/'");+ assertEquals("select id, name from sales", u.selectQuery());+ assertEquals("warehouse", u.bucket());+ assertEquals("out/", u.prefix());+ assertEquals("|", u.delimiter());+ assertFalse(u.csv());+ assertFalse(u.gzip());+ assertFalse(u.header());+ assertFalse(u.manifest());+ assertFalse(u.allowOverwrite());+ assertTrue(u.parallel());+ assertEquals(0L, u.maxFileSizeBytes());+ assertNull(u.nullAs());+ }++ @Test+ void parsesUnloadOptions() {+ CopyStatementParser.S3Unload u = unload(+ "UNLOAD ('select * from t') TO 's3://b/p/' "+ + "FORMAT AS CSV DELIMITER ',' HEADER GZIP ADDQUOTES MANIFEST "+ + "ALLOWOVERWRITE PARALLEL OFF NULL AS 'nil' MAXFILESIZE 5 MB");+ assertTrue(u.csv());+ assertEquals(",", u.delimiter());+ assertTrue(u.header());+ assertTrue(u.gzip());+ assertTrue(u.addQuotes());+ assertTrue(u.manifest());+ assertTrue(u.allowOverwrite());+ assertFalse(u.parallel());+ assertEquals("nil", u.nullAs());+ assertEquals(5L * 1024 * 1024, u.maxFileSizeBytes());+ }++ @Test+ void unloadDefaultsCsvDelimiterToComma() {+ assertEquals(",", unload("UNLOAD ('select 1') TO 's3://b/p/' CSV").delimiter());+ }++ @Test+ void unloadParsesMaxFileSizeUnits() {+ assertEquals(10L * 1024 * 1024, unload("UNLOAD ('select 1') TO 's3://b/p/' MAXFILESIZE 10 MB").maxFileSizeBytes());+ assertEquals(1024L * 1024 * 1024 / 8, unload("UNLOAD ('select 1') TO 's3://b/p/' MAXFILESIZE 0.125 GB").maxFileSizeBytes());+ assertEquals(4096L, unload("UNLOAD ('select 1') TO 's3://b/p/' MAXFILESIZE 4096").maxFileSizeBytes());+ }++ @Test+ void unloadRejectsMaxFileSizeAboveTheBufferingCeiling() {+ // The simulator buffers a file at a time, so a per-file size it cannot hold fails open.+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' MAXFILESIZE 2 GB"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' MAXFILESIZE 500 MB"));+ }++ @Test+ void unloadUnescapesDoubledSingleQuotesInSelect() {+ assertEquals("select 'x' from t",+ unload("UNLOAD ('select ''x'' from t') TO 's3://b/p/'").selectQuery());+ }++ @Test+ void unloadAcceptsWithCteSelect() {+ assertNotNull(unload("UNLOAD ('with c as (select 1) select * from c') TO 's3://b/p/'"));+ }++ @Test+ void unloadRejectsSubqueryBreakoutAttempts() {+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1) TO PROGRAM ''id'' --') TO 's3://b/p/'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1; drop table t') TO 's3://b/p/'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select $$x$$') TO 's3://b/p/'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1)') TO 's3://b/p/'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('delete from t') TO 's3://b/p/'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1 /* c */') TO 's3://b/p/'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select E''breakout''') TO 's3://b/p/'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select U&''breakout''') TO 's3://b/p/'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select \\1') TO 's3://b/p/'"));+ }++ @Test+ void unloadRejectsUnsupportedOptions() {+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' PARQUET"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' ENCRYPTED"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' IAM_ROLE 'arn:aws:iam::0:role/r'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' REGION 'us-west-2'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' EXTENSION 'csv'"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' ZSTD"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' PARTITION BY (dt)"));+ }++ @Test+ void unloadRejectsDuplicateOptionsAndUnknownTokens() {+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' GZIP GZIP"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/' FOO"));+ assertNull(CopyStatementParser.parse("UNLOAD ('select 1') TO 's3://b/p/'; SELECT 1"));+ }++ @Test+ void unloadQuotedKeywordInValueDoesNotFlipParsing() {+ CopyStatementParser.S3Unload u = unload("UNLOAD ('select 1') TO 's3://b/p/' NULL AS 'csv'");+ assertFalse(u.csv());+ assertEquals("csv", u.nullAs());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.javaindex a0f396f648..170dd4528c 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxyTest.java@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.services.acm.CertificateGenerator;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;+import io.github.hectorvent.floci.services.s3.S3Service;import org.junit.jupiter.api.AfterEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.io.TempDir;@@ -12,6 +13,7 @@ import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.lang.reflect.Field;+import java.net.InetSocketAddress;import java.net.ServerSocket;import java.net.Socket;import java.nio.charset.StandardCharsets;@@ -68,7 +70,7 @@ class RedshiftAuthProxyTest {proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),"admin", "Secret123", "dev",mock(RdsSigV4Validator.class), realTls(), (user, pw) -> true,- mock(io.github.hectorvent.floci.services.s3.S3Service.class));+ mock(S3Service.class));proxy.start(proxyPort);try (Socket client = new Socket("localhost", proxyPort)) {@@ -122,7 +124,7 @@ class RedshiftAuthProxyTest {proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),"admin", "Secret123", "dev",mock(RdsSigV4Validator.class), realTls(), (user, pw) -> true,- mock(io.github.hectorvent.floci.services.s3.S3Service.class));+ mock(S3Service.class));proxy.start(proxyPort);// Connect, then drop without ever sending a startup packet.@@ -141,7 +143,7 @@ class RedshiftAuthProxyTest {// yet; free it shortly after so the retrying bind can finally take it.ServerSocket squatter = new ServerSocket();squatter.setReuseAddress(true);- squatter.bind(new java.net.InetSocketAddress(proxyPort));+ squatter.bind(new InetSocketAddress(proxyPort));Thread.ofVirtual().start(() -> {try {Thread.sleep(200);@@ -153,7 +155,7 @@ class RedshiftAuthProxyTest {proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),"admin", "Secret123", "dev",mock(RdsSigV4Validator.class), realTls(), (user, pw) -> true,- mock(io.github.hectorvent.floci.services.s3.S3Service.class));+ mock(S3Service.class));proxy.start(proxyPort); // must not throw despite the port being busy at firstassertTrue(portAccepts(proxyPort), "proxy never bound the port after the squatter released it");@@ -166,7 +168,7 @@ class RedshiftAuthProxyTest {proxy = new RedshiftAuthProxy("111111111111:c1", "localhost", fakeBackend.getLocalPort(),"admin", "old", "dev",mock(RdsSigV4Validator.class), realTls(), (user, pw) -> true,- mock(io.github.hectorvent.floci.services.s3.S3Service.class));+ mock(S3Service.class));proxy.start(proxyPort);proxy.updateMasterPassword("rotated");diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.javaindex a7fdf5c879..2ab9e7b92e 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java@@ -12,6 +12,7 @@ import java.io.OutputStream;import java.net.ServerSocket;import java.net.Socket;import java.nio.charset.StandardCharsets;+import java.util.List;import java.util.concurrent.atomic.AtomicReference;import static org.junit.jupiter.api.Assertions.assertArrayEquals;@@ -212,4 +213,52 @@ class RedshiftInterceptingBridgeTest {assertEquals('Q', forwarded.type());assertTrue(forwarded.getSql().contains("s3://wh/k"), forwarded.getSql());}++ @Test+ void unloadQueryIsDispatchedToTheSimulator() throws Exception {+ startBridge();+ Mockito.when(s3Stub.listObjectsWithPrefixes(Mockito.eq("b"), Mockito.eq("out/"),+ Mockito.isNull(), Mockito.anyInt(), Mockito.any(), Mockito.any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));+ AtomicReference<byte[]> put = new AtomicReference<>();+ Mockito.when(s3Stub.putObject(Mockito.eq("b"), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))+ .thenAnswer(inv -> {+ put.compareAndSet(null, inv.getArgument(2));+ return null;+ });++ // Fake backend: expect the fabricated COPY (...) TO STDOUT and play one row.+ Thread be = Thread.ofVirtual().start(() -> {+ try {+ PostgresWireDecoder in = new PostgresWireDecoder(testBackendEnd.getInputStream());+ PostgresWireDecoder.FrontendMessage q = in.nextMessage();+ assertTrue(q.getSql().contains("TO STDOUT"), q.getSql());+ OutputStream out = testBackendEnd.getOutputStream();+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});+ byte[] row = "1|a\n".getBytes(StandardCharsets.US_ASCII);+ out.write('d');+ out.write(new byte[]{(byte) (0), 0, 0, (byte) (4 + row.length)});+ out.write(row);+ out.write(new byte[]{'c', 0, 0, 0, 4});+ byte[] tag = "COPY 1\0".getBytes(StandardCharsets.US_ASCII);+ out.write('C');+ out.write(new byte[]{0, 0, 0, (byte) (4 + tag.length)});+ out.write(tag);+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});+ out.flush();+ } catch (IOException e) {+ LOG.debugv(e, "fake backend ended");+ }+ });++ OutputStream clientOut = testClientEnd.getOutputStream();+ clientOut.write(PostgresWireDecoder.encodeQuery("UNLOAD ('select a,b from t') TO 's3://b/out/'"));+ clientOut.flush();++ PostgresWireDecoder fromPump = new PostgresWireDecoder(testClientEnd.getInputStream());+ assertEquals('C', fromPump.nextMessage().type());+ assertEquals('Z', fromPump.nextMessage().type());+ be.join();+ assertNotNull(put.get(), "the simulator should have written one S3 object");+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.javaindex d62c6658f8..dcbc75bd2c 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManagerTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.redshift.proxy;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;+import io.github.hectorvent.floci.services.s3.S3Service;import org.junit.jupiter.api.Test;import java.io.IOException;@@ -28,7 +29,7 @@ class RedshiftProxyManagerTest {private RedshiftProxyManager newManager() {return new RedshiftProxyManager(mock(RdsSigV4Validator.class), mock(RdsProxyTlsCertificates.class),- mock(io.github.hectorvent.floci.services.s3.S3Service.class));+ mock(S3Service.class));}private static void start(RedshiftProxyManager manager, String key, int proxyPort) {diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.javaindex 28b9a8aeaa..9570a4158d 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java@@ -7,6 +7,7 @@ import org.junit.jupiter.api.AfterEach;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;import java.io.IOException;import java.io.OutputStream;@@ -14,12 +15,17 @@ import java.net.ServerSocket;import java.net.Socket;import java.net.SocketTimeoutException;import java.nio.charset.StandardCharsets;+import java.util.ArrayList;import java.util.List;+import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.atomic.AtomicReference;+import java.util.zip.GZIPInputStream;import java.util.zip.GZIPOutputStream;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -108,6 +114,598 @@ class S3CopySimulatorTest {return new PostgresWireDecoder(testBackend.getInputStream()).nextMessage();}+ /** Play a PostgreSQL backend for a fabricated COPY (...) TO STDOUT: 'H', N 'd' rows, 'c', 'C', 'Z'. */+ private void playUnloadBackend(String... rows) throws IOException {+ OutputStream out = testBackend.getOutputStream();+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());+ PostgresWireDecoder.FrontendMessage q = in.nextMessage();+ assertEquals('Q', q.type());+ assertTrue(q.getSql().contains("TO STDOUT"), q.getSql());+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0}); // CopyOutResponse+ for (String row : rows) {+ byte[] b = row.getBytes(StandardCharsets.US_ASCII);+ out.write('d');+ out.write(intBytes(4 + b.length));+ out.write(b);+ }+ out.write(new byte[]{'c', 0, 0, 0, 4}); // CopyDone+ byte[] tag = "COPY 2\0".getBytes(StandardCharsets.US_ASCII);+ out.write('C');+ out.write(intBytes(4 + tag.length));+ out.write(tag);+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});+ out.flush();+ }++ private CopyStatementParser.S3Unload unloadSpec(String bucket, String prefix,+ boolean gzip, boolean manifest, boolean allowOverwrite, boolean parallel, long maxFileSize) {+ return new CopyStatementParser.S3Unload("select a,b from t", bucket, prefix,+ "|", false, gzip, false, false, null, manifest, allowOverwrite, parallel, maxFileSize);+ }++ @Test+ void unloadWritesSingleObjectAndForwardsCommandComplete() throws Exception {+ Map<String, byte[]> written = new ConcurrentHashMap<>();+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {+ written.put(inv.getArgument(1), inv.getArgument(2));+ return null;+ });+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);+ Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n"));++ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ assertTrue(handled);+ assertEquals(1, written.size());+ assertTrue(written.containsKey("out/0000_part_00"), written.keySet().toString());+ assertEquals("1|alice\n2|bob\n", new String(written.get("out/0000_part_00"), StandardCharsets.US_ASCII));++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ assertEquals('C', in.nextMessage().type());+ assertEquals('Z', in.nextMessage().type());+ }++ @Test+ void unloadRepeatsTheHeaderRowInEverySlice() throws Exception {+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 4; // one data row per slice+ try {+ Map<String, byte[]> written = new ConcurrentHashMap<>();+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {+ written.put(inv.getArgument(1), inv.getArgument(2));+ return null;+ });+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));++ CopyStatementParser.S3Unload spec = new CopyStatementParser.S3Unload(+ "select a,b from t", "wh", "out/", "|", true, false, true, false, null,+ false, false, true, 0);+ // The fake backend stands in for PostgreSQL: the first CopyData frame is the header row.+ Thread backend = backendThread(() -> playUnloadBackend("h1|h2\n", "1|a\n", "2|b\n", "3|c\n"));+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ assertEquals(3, written.size(), written.keySet().toString());+ for (Map.Entry<String, byte[]> e : written.entrySet()) {+ String body = new String(e.getValue(), StandardCharsets.US_ASCII);+ assertTrue(body.startsWith("h1|h2\n"), e.getKey() + " -> " + body);+ }+ } finally {+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;+ }+ }++ @Test+ void unloadSplitsIntoMultipleObjectsOnNewlineBoundaries() throws Exception {+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 6; // force a split after each row+ try {+ Map<String, byte[]> written = new ConcurrentHashMap<>();+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {+ written.put(inv.getArgument(1), inv.getArgument(2));+ return null;+ });+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);+ Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n", "3|carol\n"));+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ assertTrue(written.containsKey("out/0000_part_00"));+ assertTrue(written.containsKey("out/0001_part_00"));+ assertTrue(written.containsKey("out/0002_part_00"));+ assertEquals(3, written.size(), written.keySet().toString());+ StringBuilder all = new StringBuilder();+ written.entrySet().stream().sorted(Map.Entry.comparingByKey())+ .forEach(e -> all.append(new String(e.getValue(), StandardCharsets.US_ASCII)));+ assertEquals("1|alice\n2|bob\n3|carol\n", all.toString());+ } finally {+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;+ }+ }++ @Test+ void unloadParallelOffUsesThreeDigitKeyNames() throws Exception {+ Map<String, byte[]> written = new ConcurrentHashMap<>();+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {+ written.put(inv.getArgument(1), inv.getArgument(2));+ return null;+ });+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, false, 0);+ Thread backend = backendThread(() -> playUnloadBackend("1|a\n"));+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ assertTrue(written.containsKey("out/000"), written.keySet().toString());+ }++ @Test+ void unloadGzipAppendsGzSuffixAndWritesDecompressibleBytes() throws Exception {+ Map<String, byte[]> written = new ConcurrentHashMap<>();+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {+ written.put(inv.getArgument(1), inv.getArgument(2));+ return null;+ });+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", true, false, false, true, 0);+ Thread backend = backendThread(() -> playUnloadBackend("1|a\n", "2|b\n"));+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ assertTrue(written.containsKey("out/0000_part_00.gz"), written.keySet().toString());+ byte[] gz = written.get("out/0000_part_00.gz");+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(gz))) {+ in.transferTo(out);+ }+ assertEquals("1|a\n2|b\n", out.toString(StandardCharsets.US_ASCII));+ }++ @Test+ void unloadManifestListsEveryDataObject() throws Exception {+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 8;+ try {+ Map<String, byte[]> written = new ConcurrentHashMap<>();+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {+ written.put(inv.getArgument(1), inv.getArgument(2));+ return null;+ });+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, false, true, 0);+ Thread backend = backendThread(() -> playUnloadBackend("1|alice\n", "2|bob\n", "3|carol\n"));+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ String manifest = new String(written.get("out/manifest"), StandardCharsets.UTF_8);+ assertTrue(manifest.contains("s3://wh/out/0000_part_00"), manifest);+ assertTrue(manifest.contains("s3://wh/out/0001_part_00"), manifest);+ assertTrue(manifest.contains("\"content_length\":"), manifest);+ } finally {+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;+ }+ }++ @Test+ void unloadIntoNonEmptyPrefixWithoutAllowOverwriteErrorsAndSkipsSelect() throws Exception {+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(+ List.of(new S3Object("wh", "out/old", new byte[]{1}, "text/plain")), List.of(), false, null));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);+ // No backend thread: the select must never run.+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ assertTrue(handled);++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ assertEquals('E', in.nextMessage().type());+ assertEquals('Z', in.nextMessage().type());+ }++ @Test+ void unloadForwardsBackendErrorWhenSelectIsInvalid() throws Exception {+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);++ Thread backend = backendThread(() -> {+ OutputStream out = testBackend.getOutputStream();+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());+ assertEquals('Q', in.nextMessage().type());+ byte[] err = "SERROR\0C42703\0Mcolumn \"nope\" does not exist\0\0".getBytes(StandardCharsets.US_ASCII);+ out.write('E');+ out.write(intBytes(4 + err.length));+ out.write(err);+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});+ out.flush();+ });++ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ assertEquals('E', in.nextMessage().type());+ assertEquals('Z', in.nextMessage().type());+ testClient.setSoTimeout(400);+ assertThrows(SocketTimeoutException.class, () -> testClient.getInputStream().read());+ }++ @Test+ void unloadAbortsWhenResultExceedsTotalCeiling() throws Exception {+ long saved = S3CopySimulator.UNLOAD_MAX_TOTAL_BYTES;+ S3CopySimulator.UNLOAD_MAX_TOTAL_BYTES = 16;+ try {+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 1_000_000);++ Thread backend = backendThread(() -> {+ OutputStream out = testBackend.getOutputStream();+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());+ assertEquals('Q', in.nextMessage().type());+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});+ byte[] big = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n".getBytes(StandardCharsets.US_ASCII);+ out.write('d');+ out.write(intBytes(4 + big.length));+ out.write(big);+ out.flush();+ try {+ out.write(new byte[]{'c', 0, 0, 0, 4});+ out.flush();+ } catch (IOException ignored) {+ // Backend socket is closed immediately by simulator on ceiling exceed+ }+ });++ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ PostgresWireDecoder.FrontendMessage err = in.nextMessage();+ assertEquals('E', err.type());+ assertTrue(new String(err.body(), StandardCharsets.US_ASCII).contains("54000"));+ assertEquals('Z', in.nextMessage().type());+ assertNull(in.nextMessage(), "Client must receive EOF after ReadyForQuery on connection close");+ } finally {+ S3CopySimulator.UNLOAD_MAX_TOTAL_BYTES = saved;+ }+ }++ @Test+ void unloadInTransactionBlockReportsFailedTransactionStatusOnError() throws Exception {+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(+ List.of(new S3Object("wh", "out/old", new byte[]{1}, "text/plain")), List.of(), false, null));+ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);++ Thread backend = backendThread(() -> {+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());+ PostgresWireDecoder.FrontendMessage q = in.nextMessage();+ assertEquals('Q', q.type());+ assertTrue(q.getSql().contains("FLOCI_ABORT_TX"));+ OutputStream out = testBackend.getOutputStream();+ byte[] err = "SERROR\0C42601\0Msyntax error\0\0".getBytes(StandardCharsets.US_ASCII);+ out.write('E');+ out.write(intBytes(4 + err.length));+ out.write(err);+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'E'});+ out.flush();+ });++ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'T');+ joinBackend(backend);++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ assertEquals('E', in.nextMessage().type());+ PostgresWireDecoder.FrontendMessage z = in.nextMessage();+ assertEquals('Z', z.type());+ assertEquals('E', (char) z.body()[0]);+ }++ @Test+ void unloadStreamingFailureDrainsBackendAndSendsSingleError() throws Exception {+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 8; // Force flush on first row+ try {+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));+ when(s3.putObject(eq("wh"), any(), any(), any(), any()))+ .thenThrow(new RuntimeException("simulated S3 storage failure"));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);++ Thread backend = backendThread(() -> {+ OutputStream out = testBackend.getOutputStream();+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());+ assertEquals('Q', in.nextMessage().type());+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});+ // Send multiple rows so the backend has pending messages when putObject throws mid-stream on the first slice+ byte[] row1 = "1|alice\n".getBytes(StandardCharsets.US_ASCII);+ out.write('d');+ out.write(intBytes(4 + row1.length));+ out.write(row1);+ byte[] row2 = "2|bob\n".getBytes(StandardCharsets.US_ASCII);+ out.write('d');+ out.write(intBytes(4 + row2.length));+ out.write(row2);+ out.write(new byte[]{'c', 0, 0, 0, 4});+ byte[] tag = "COPY 2\0".getBytes(StandardCharsets.US_ASCII);+ out.write('C');+ out.write(intBytes(4 + tag.length));+ out.write(tag);+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});+ out.flush();+ });++ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ assertTrue(handled);+ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ assertEquals('E', in.nextMessage().type());+ assertEquals('Z', in.nextMessage().type());+ testClient.setSoTimeout(400);+ assertThrows(SocketTimeoutException.class, () -> testClient.getInputStream().read(),+ "no leftover or duplicate messages should reach client");+ } finally {+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;+ }+ }++ @Test+ void unloadAbortsWhenMemoryBudgetExhaustedReports53400() throws Exception {+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);++ S3CopySimulator.UNLOAD_HEAP_MIB.acquire(192);+ try {+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ assertTrue(handled);++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ PostgresWireDecoder.FrontendMessage err = in.nextMessage();+ assertEquals('E', err.type());+ String errBody = new String(err.body(), StandardCharsets.UTF_8);+ assertTrue(errBody.contains("53400"), errBody);+ assertTrue(errBody.contains("memory budget exhausted"), errBody);+ assertEquals('Z', in.nextMessage().type());+ } finally {+ S3CopySimulator.UNLOAD_HEAP_MIB.release(192);+ }+ }++ @Test+ void unloadFabricatedCopySqlMatchesFormattingOptions() throws Exception {+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));+ CopyStatementParser.S3Unload spec = new CopyStatementParser.S3Unload(+ "select a from t", "wh", "out/", "\t", true, false, false, true, "\\N", false, true, true, 0);++ AtomicReference<String> seenSql = new AtomicReference<>();+ Thread backend = backendThread(() -> {+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());+ PostgresWireDecoder.FrontendMessage q = in.nextMessage();+ seenSql.set(q.getSql());+ OutputStream out = testBackend.getOutputStream();+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});+ out.write(new byte[]{'c', 0, 0, 0, 4});+ byte[] tag = "COPY 0\0".getBytes(StandardCharsets.US_ASCII);+ out.write('C');+ out.write(intBytes(4 + tag.length));+ out.write(tag);+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});+ out.flush();+ });++ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ String sql = seenSql.get();+ assertNotNull(sql);+ assertTrue(sql.contains("FORMAT csv"), sql);+ assertTrue(sql.contains("DELIMITER '\t'"), sql);+ assertTrue(sql.contains("HEADER true"), sql);+ assertTrue(sql.contains("FORCE_QUOTE *"), sql);+ assertTrue(sql.contains("NULL '\\N'"), sql);+ }++ @Test+ void unloadZeroRowsEmitsSingleEmptyObject() throws Exception {+ Map<String, byte[]> written = new ConcurrentHashMap<>();+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {+ written.put(inv.getArgument(1), inv.getArgument(2));+ return null;+ });+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);+ Thread backend = backendThread(() -> {+ OutputStream out = testBackend.getOutputStream();+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());+ assertEquals('Q', in.nextMessage().type());+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});+ out.write(new byte[]{'c', 0, 0, 0, 4});+ byte[] tag = "COPY 0\0".getBytes(StandardCharsets.US_ASCII);+ out.write('C');+ out.write(intBytes(4 + tag.length));+ out.write(tag);+ out.write(new byte[]{'Z', 0, 0, 0, 5, 'I'});+ out.flush();+ });++ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ assertTrue(handled);+ assertEquals(1, written.size());+ assertTrue(written.containsKey("out/0000_part_00"));+ assertEquals(0, written.get("out/0000_part_00").length);+ }++ @Test+ void unloadBucketNotFoundSurfacesNotFoundMessageInsteadOfAccessDenied() throws Exception {+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenThrow(new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ assertTrue(handled);++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ PostgresWireDecoder.FrontendMessage err = in.nextMessage();+ assertEquals('E', err.type());+ String body = new String(err.body(), StandardCharsets.UTF_8);+ assertTrue(body.contains("not found"), body);+ assertFalse(body.contains("access denied"), body);+ assertEquals('Z', in.nextMessage().type());+ }++ @Test+ void unloadManifestFailureCleansUpWrittenDataObjects() throws Exception {+ List<String> deleted = new ArrayList<>();+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {+ String key = inv.getArgument(1);+ if (key.endsWith("manifest")) {+ throw new RuntimeException("disk full writing manifest");+ }+ return null;+ });+ when(s3.deleteObject(eq("wh"), any())).thenAnswer(inv -> {+ deleted.add(inv.getArgument(1));+ return null;+ });++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, false, true, 0);+ Thread backend = backendThread(() -> playUnloadBackend("1|a\n"));+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ assertEquals(1, deleted.size());+ assertEquals("out/0000_part_00", deleted.get(0));++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ assertEquals('E', in.nextMessage().type());+ assertEquals('Z', in.nextMessage().type());+ }++ @Test+ void unloadManifestFailureUnderAllowOverwriteLeavesEveryDataObjectInPlace() throws Exception {+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 4; // one data row per slice+ try {+ List<String> deleted = new ArrayList<>();+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));+ when(s3.putObject(eq("wh"), any(), any(), any(), any())).thenAnswer(inv -> {+ if (((String) inv.getArgument(1)).endsWith("manifest")) {+ throw new RuntimeException("disk full writing manifest");+ }+ return null;+ });+ when(s3.deleteObject(eq("wh"), any())).thenAnswer(inv -> {+ deleted.add(inv.getArgument(1));+ return null;+ });++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, true, true, 0);+ Thread backend = backendThread(() -> playUnloadBackend("1|a\n", "2|b\n"));+ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ // Under ALLOWOVERWRITE a written key may have replaced pre-existing (or another+ // operation's) data, so cleanup deletes nothing and the partial result is left behind.+ assertTrue(deleted.isEmpty(), "ALLOWOVERWRITE cleanup must not delete any object, got " + deleted);++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ assertEquals('E', in.nextMessage().type());+ assertEquals('Z', in.nextMessage().type());+ } finally {+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;+ }+ }++ @Test+ void unloadWithoutAllowOverwriteFailsIfListBucketDenied() throws Exception {+ doThrow(new AwsException("AccessDenied", "Access Denied", 403))+ .when(s3).authorizeAnonymousListBucket(eq("wh"));++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, false, false, true, 0);+ boolean handled = S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ assertTrue(handled);++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ PostgresWireDecoder.FrontendMessage err = in.nextMessage();+ assertEquals('E', err.type());+ String body = new String(err.body(), StandardCharsets.UTF_8);+ assertTrue(body.contains("42501"), body);+ assertTrue(body.contains("access denied"), body);+ assertEquals('Z', in.nextMessage().type());+ }++ @Test+ void unloadBackendEofMidStreamWithManifestCleansUpWrittenDataObjects() throws Exception {+ long saved = S3CopySimulator.UNLOAD_TARGET_FILE_BYTES;+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = 6;+ try {+ List<String> deleted = new ArrayList<>();+ when(s3.listObjectsWithPrefixes(eq("wh"), eq("out/"), isNull(), anyInt(), any(), any()))+ .thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));+ when(s3.deleteObject(eq("wh"), any())).thenAnswer(inv -> {+ deleted.add(inv.getArgument(1));+ return null;+ });++ CopyStatementParser.S3Unload spec = unloadSpec("wh", "out/", false, true, false, true, 0);++ Thread backend = backendThread(() -> {+ OutputStream out = testBackend.getOutputStream();+ PostgresWireDecoder in = new PostgresWireDecoder(testBackend.getInputStream());+ assertEquals('Q', in.nextMessage().type());+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});+ byte[] row = "1|alice\n".getBytes(StandardCharsets.US_ASCII);+ out.write('d');+ out.write(intBytes(4 + row.length));+ out.write(row);+ out.flush();+ testBackend.close();+ });++ S3CopySimulator.runUnload(simClient, simBackend, spec, s3, 'I');+ joinBackend(backend);++ assertEquals(1, deleted.size());+ assertEquals("out/0000_part_00", deleted.get(0));++ PostgresWireDecoder in = new PostgresWireDecoder(testClient.getInputStream());+ PostgresWireDecoder.FrontendMessage err = in.nextMessage();+ assertEquals('E', err.type());+ String body = new String(err.body(), StandardCharsets.UTF_8);+ assertTrue(body.contains("backend closed mid-stream"), body);+ assertEquals('Z', in.nextMessage().type());+ } finally {+ S3CopySimulator.UNLOAD_TARGET_FILE_BYTES = saved;+ }+ }+/** Play a minimal happy-path PostgreSQL backend: 'G' then, after CopyDone, 'C' and 'Z'. */private void playHappyBackend(ByteArrayOutputStream capturedCopyData) throws IOException {OutputStream out = testBackend.getOutputStream();diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.javaindex 77d3be2145..7d6d554b38 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java@@ -1149,4 +1149,18 @@ class S3ServiceTest {"configuration config-" + i + " was lost by a concurrent put");}}++ @Test+ void authorizeAnonymousPutObjectIsANoOpWhenEnforceAuthIsOff() {+ // Default test config has FLOCI_SERVICES_S3_ENFORCE_AUTH unset/false.+ s3Service.createBucket("anon-put-bucket", "us-east-1");+ assertDoesNotThrow(() -> s3Service.authorizeAnonymousPutObject("anon-put-bucket", "some/key"));+ }++ @Test+ void authorizeAnonymousDeleteObjectIsANoOpWhenEnforceAuthIsOff() {+ s3Service.createBucket("anon-del-bucket", "us-east-1");+ assertDoesNotThrow(() -> s3Service.authorizeAnonymousDeleteObject("anon-del-bucket", "some/key"));+ }}+