Light, fluffy, and always free - The AWS Local Emulator alternative
feat(redshift): intercept Extended Query protocol (#3474)
Vũ Thế Huye9454af85505
diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.javaindex a0da478d40..e5eab8f8b5 100644--- a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java@@ -6,6 +6,7 @@ import org.junit.jupiter.api.Order;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.TestMethodOrder;import software.amazon.awssdk.core.exception.SdkClientException;+import software.amazon.awssdk.core.sync.RequestBody;import software.amazon.awssdk.services.redshift.RedshiftClient;import software.amazon.awssdk.services.redshift.model.Cluster;import software.amazon.awssdk.services.redshift.model.CreateClusterRequest;@@ -21,11 +22,14 @@ import software.amazon.awssdk.services.s3.model.CreateBucketRequest;import software.amazon.awssdk.services.s3.model.GetObjectRequest;import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;+import software.amazon.awssdk.services.s3.model.PutObjectRequest;import java.net.ConnectException;import java.nio.charset.StandardCharsets;import java.sql.Connection;import java.sql.DriverManager;+import java.sql.PreparedStatement;+import java.sql.ResultSet;import java.sql.Statement;import java.time.Duration;@@ -121,19 +125,33 @@ public class RedshiftTest {.clusterIdentifier("test-cluster").build()).clusters().get(0);String jdbcUrl = "jdbc:postgresql://" + cluster.endpoint().address() + ":"- + cluster.endpoint().port() + "/dev?preferQueryMode=simple";+ + cluster.endpoint().port() + "/dev";S3Client s3 = TestFixtures.s3Client();String bucket = "redshift-unload-compat";s3.createBucket(CreateBucketRequest.builder().bucket(bucket).build());+ s3.putObject(PutObjectRequest.builder().bucket(bucket).key("in/data.txt").contentType("text/plain").build(),+ RequestBody.fromString("1|alice\n2|bob\n"));try (Connection conn = DriverManager.getConnection(jdbcUrl, "admin", "Password123");Statement st = conn.createStatement()) {st.execute("DROP TABLE IF EXISTS unload_src");- st.execute("CREATE TABLE unload_src (id int, name text)");- st.execute("INSERT INTO unload_src VALUES (1, 'alice'), (2, 'bob')");- st.execute("UNLOAD ('select id, name from unload_src order by id') "- + "TO 's3://redshift-unload-compat/out/'");+ try (PreparedStatement ddl = conn.prepareStatement("CREATE TABLE unload_src (id int, name text)")) {+ ddl.execute();+ }+ try (PreparedStatement copy = conn.prepareStatement(+ "COPY unload_src FROM 's3://redshift-unload-compat/in/data.txt'")) {+ copy.execute();+ }+ try (PreparedStatement unload = conn.prepareStatement(+ "UNLOAD ('select id, name from unload_src order by id') "+ + "TO 's3://redshift-unload-compat/out/' ALLOWOVERWRITE")) {+ unload.execute();+ }+ try (ResultSet rows = st.executeQuery("SELECT count(*) FROM unload_src")) {+ assertTrue(rows.next());+ assertEquals(2, rows.getInt(1));+ }}ListObjectsV2Response listing = s3.listObjectsV2(ListObjectsV2Request.builder()diff --git a/docs/services/redshift.md b/docs/services/redshift.mdindex 32c24ca77f..4aec3461c1 100644--- a/docs/services/redshift.md+++ b/docs/services/redshift.md@@ -141,7 +141,7 @@ print(cluster["Cluster"]["Endpoint"])## SQL Interceptor-Floci's Redshift auth proxy inspects frontend queries on the PostgreSQL wire protocol (Simple Query `'Q'` protocol) and rewrites common Redshift-specific table DDL so it runs on the plain PostgreSQL backend.+Floci's Redshift auth proxy inspects frontend queries on the PostgreSQL wire protocol and rewrites common Redshift-specific table DDL so it runs on the plain PostgreSQL backend.### DDL compatibility@@ -170,12 +170,15 @@ order) through its own S3 service and streams the rows into the backing PostgreSrecognized: the statement is forwarded unchanged and PostgreSQL returns its own error.- A multi-statement query whose COPY is followed by another statement is not intercepted; send theCOPY on its own.-- Extended Query protocol COPY (a JDBC `PreparedStatement`, or pgjdbc's default- `preferQueryMode=extended`) is not intercepted. Use `preferQueryMode=simple`.+- Extended Query COPY is supported when the complete statement is present in `Parse` and has no+ bind parameters. Zero-parameter JDBC `PreparedStatement` calls therefore work with pgjdbc's+ default extended mode. Statements containing bind parameters are forwarded unchanged.### Limitations-- Emulation runs on the **Simple Query protocol** (`'Q'`) only. Extended Query protocol statements (`Parse`/`Bind`/`Execute`) pass through untouched, including anything a JDBC `PreparedStatement` sends, and, with the pgjdbc default `preferQueryMode=extended`, plain `Statement` calls too. Connect with `preferQueryMode=simple` to exercise the interceptor from JDBC.+- DDL rewriting works in both Simple Query (`'Q'`) and Extended Query (`Parse`) flows. COPY and+ UNLOAD interception in Extended Query is limited to zero-parameter statements fully present in+ `Parse`; parameterized statements fail open to PostgreSQL.- The rewrite is textual (regex-based). It masks single-quoted string literals first, so `DEFAULT` / `CHECK` string values are safe, but it is **not** comment-aware and does not recognize escape strings (`E'...'`): an apostrophe inside a `--` or `/* */` comment can make the rewrite skip a Redshift clause. That fails safe: the statement then reaches PostgreSQL, which returns its own syntax error, but avoid apostrophes-in-comments in `CREATE TABLE` / `ALTER TABLE`.- A `rewrite` failure or any statement the interceptor does not recognize is forwarded unmodified (fail-open); PostgreSQL then rejects the Redshift-only syntax itself.- Simple Query ('Q') messages larger than 16 MiB bypass the interceptor and stream through verbatim without heap buffering; non-query traffic also streams through with no size limit.@@ -183,8 +186,8 @@ order) through its own S3 service and streams the rows into the backing PostgreS### UNLOAD to S3-`UNLOAD ('<select-statement>') TO 's3://<bucket>/<prefix>' [options]` sent over the-Simple Query protocol runs the select on the backing PostgreSQL container and writes+`UNLOAD ('<select-statement>') TO 's3://<bucket>/<prefix>' [options]` runs the select on the+backing PostgreSQL container and writesthe result to S3 as one or more objects under `<prefix>`.- Framing defaults to pipe-delimited text; `FORMAT CSV` (or `CSV`) switches to CSV@@ -214,7 +217,8 @@ the result to S3 as one or more objects under `<prefix>`.- Any other option (`PARQUET`, `ENCRYPTED`, `REGION`, `IAM_ROLE` / `CREDENTIALS`,`ZSTD`, `EXTENSION`, `CLEANPATH`, `PARTITION`, and so on) is not intercepted; thestatement is forwarded and PostgreSQL reports its own error.-- Extended Query protocol UNLOAD (a JDBC `PreparedStatement`) is not intercepted.+- Extended Query UNLOAD is supported when the complete statement is present in `Parse` and has no+ bind parameters. Parameterized statements are forwarded unchanged.## Out of Scopediff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.javanew file mode 100644index 0000000000..71ec88f443--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java@@ -0,0 +1,295 @@+package io.github.hectorvent.floci.services.redshift.proxy;++import java.util.ArrayDeque;+import java.util.HashMap;+import java.util.Map;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.locks.Condition;+import java.util.concurrent.locks.ReentrantLock;++final class BackendResponseCoordinator {++ private final ExtendedQuerySession session;+ private final ReentrantLock lock = new ReentrantLock(true);+ private final Condition changed = lock.newCondition();+ private final ArrayDeque<PendingOperation> pending = new ArrayDeque<>();+ private final Map<Long, GateResult> resolvedGates = new HashMap<>();+ private long nextSequence = 1;+ private boolean discardingUntilSync;+ private boolean closed;+ private char lastReadyStatus = 'I';++ BackendResponseCoordinator(ExtendedQuerySession session) {+ this.session = session;+ }++ Ticket register(Operation operation, ExtendedQuerySession.Mutation mutation) {+ lock.lock();+ try {+ Ticket ticket = new Ticket(nextSequence++, operation);+ if (discardingUntilSync && operation != Operation.SYNC) {+ if (mutation != null) {+ session.rejectFrom(mutation);+ }+ if (operation == Operation.EXECUTE) {+ resolvedGates.put(ticket.sequence(), GateResult.SKIPPED_AFTER_ERROR);+ }+ } else {+ pending.addLast(new PendingOperation(ticket, mutation));+ }+ changed.signalAll();+ return ticket;+ } finally {+ lock.unlock();+ }+ }++ GateResult awaitTurn(Ticket ticket) throws InterruptedException {+ lock.lockInterruptibly();+ try {+ while (true) {+ if (closed) {+ return GateResult.CLOSED;+ }+ GateResult resolved = resolvedGates.remove(ticket.sequence());+ if (resolved != null) {+ return resolved;+ }+ PendingOperation head = pending.peekFirst();+ if (!discardingUntilSync && head != null && head.ticket().equals(ticket)) {+ return GateResult.READY;+ }+ changed.await();+ }+ } finally {+ lock.unlock();+ }+ }++ GateResult awaitExtendedExecuteTurn(Ticket ticket) throws InterruptedException {+ lock.lockInterruptibly();+ try {+ while (true) {+ if (closed) {+ return GateResult.CLOSED;+ }+ GateResult resolved = resolvedGates.remove(ticket.sequence());+ if (resolved != null) {+ return resolved;+ }+ if (!discardingUntilSync && canOwnExtendedExecute(ticket)) {+ return GateResult.READY;+ }+ changed.await();+ }+ } finally {+ lock.unlock();+ }+ }++ boolean awaitIdle(long timeout, TimeUnit unit) throws InterruptedException {+ long remaining = unit.toNanos(timeout);+ lock.lockInterruptibly();+ try {+ while ((!pending.isEmpty() || discardingUntilSync) && !closed) {+ if (remaining <= 0) {+ return false;+ }+ remaining = changed.awaitNanos(remaining);+ }+ return !closed && pending.isEmpty() && !discardingUntilSync;+ } finally {+ lock.unlock();+ }+ }++ void onBackendFrame(char type, byte[] body) {+ lock.lock();+ try {+ if (type == 'Z') {+ updateReadyStatus(body);+ }+ if (isAsynchronous(type)) {+ return;+ }++ PendingOperation head = pending.peekFirst();+ if (head == null) {+ changed.signalAll();+ return;+ }++ Operation operation = head.ticket().operation();+ if (type == 'E') {+ if (operation != Operation.SIMPLE_QUERY) {+ pending.removeFirst();+ reject(head);+ resolveSkippedGate(head);+ discardingUntilSync = true;+ discardQueuedOperationsBeforeSync();+ }+ changed.signalAll();+ return;+ }++ if (completes(operation, type)) {+ pending.removeFirst();+ confirm(head);+ if (operation == Operation.SYNC) {+ discardingUntilSync = false;+ }+ changed.signalAll();+ }+ } finally {+ lock.unlock();+ }+ }++ void completeOwnedExecute(Ticket ticket, boolean failed) {+ lock.lock();+ try {+ PendingOperation owned = pending.stream()+ .filter(operation -> operation.ticket().equals(ticket))+ .findFirst()+ .orElse(null);+ if (owned == null) {+ return;+ }+ if (ticket.operation() != Operation.EXECUTE) {+ throw new IllegalStateException("Only an Execute ticket can be completed as backend-owned");+ }+ pending.remove(owned);+ confirm(owned);+ if (failed) {+ discardingUntilSync = true;+ discardQueuedOperationsBeforeSync();+ }+ changed.signalAll();+ } finally {+ lock.unlock();+ }+ }++ boolean isDiscardingUntilSync() {+ lock.lock();+ try {+ return discardingUntilSync;+ } finally {+ lock.unlock();+ }+ }++ char lastReadyStatus() {+ lock.lock();+ try {+ return lastReadyStatus;+ } finally {+ lock.unlock();+ }+ }++ void close() {+ lock.lock();+ try {+ closed = true;+ pending.clear();+ resolvedGates.clear();+ session.clear();+ changed.signalAll();+ } finally {+ lock.unlock();+ }+ }++ private void discardQueuedOperationsBeforeSync() {+ while (!pending.isEmpty() && pending.peekFirst().ticket().operation() != Operation.SYNC) {+ PendingOperation skipped = pending.removeFirst();+ reject(skipped);+ resolveSkippedGate(skipped);+ }+ }++ private void resolveSkippedGate(PendingOperation operation) {+ if (operation.ticket().operation() == Operation.EXECUTE) {+ resolvedGates.put(operation.ticket().sequence(), GateResult.SKIPPED_AFTER_ERROR);+ }+ }++ private void updateReadyStatus(byte[] body) {+ if (body != null && body.length == 1) {+ lastReadyStatus = (char) body[0];+ if (lastReadyStatus == 'I') {+ session.transactionEnded();+ }+ }+ }++ private void confirm(PendingOperation operation) {+ if (operation.mutation() != null) {+ session.confirm(operation.mutation());+ }+ }++ private void reject(PendingOperation operation) {+ if (operation.mutation() != null) {+ session.rejectFrom(operation.mutation());+ }+ }++ private boolean canOwnExtendedExecute(Ticket ticket) {+ for (PendingOperation operation : pending) {+ if (operation.ticket().equals(ticket)) {+ return true;+ }+ if (!isExtendedExecutePreamble(operation.ticket().operation())) {+ return false;+ }+ }+ return false;+ }++ private static boolean isExtendedExecutePreamble(Operation operation) {+ return operation == Operation.PARSE+ || operation == Operation.BIND+ || operation == Operation.DESCRIBE_STATEMENT+ || operation == Operation.DESCRIBE_PORTAL;+ }++ private static boolean isAsynchronous(char type) {+ return type == 'N' || type == 'A' || type == 'S';+ }++ private static boolean completes(Operation operation, char responseType) {+ return switch (operation) {+ case SIMPLE_QUERY, SYNC -> responseType == 'Z';+ case PARSE -> responseType == '1';+ case BIND -> responseType == '2';+ case DESCRIBE_STATEMENT, DESCRIBE_PORTAL -> responseType == 'T' || responseType == 'n';+ case CLOSE -> responseType == '3';+ case EXECUTE -> responseType == 'C' || responseType == 'I' || responseType == 's';+ };+ }++ enum Operation {+ SIMPLE_QUERY,+ PARSE,+ BIND,+ DESCRIBE_STATEMENT,+ DESCRIBE_PORTAL,+ CLOSE,+ EXECUTE,+ SYNC+ }++ enum GateResult {+ READY,+ SKIPPED_AFTER_ERROR,+ CLOSED+ }++ record Ticket(long sequence, Operation operation) {+ }++ private record PendingOperation(Ticket ticket, ExtendedQuerySession.Mutation mutation) {+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySession.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySession.javanew file mode 100644index 0000000000..1e03c751af--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySession.java@@ -0,0 +1,191 @@+package io.github.hectorvent.floci.services.redshift.proxy;++import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Map;+import java.util.Objects;+import java.util.Optional;+import java.util.Set;++final class ExtendedQuerySession {++ private final Map<String, CopyStatementParser.S3Statement> statements = new LinkedHashMap<>();+ private final Map<String, String> portals = new LinkedHashMap<>();+ private final List<JournalEntry> journal = new ArrayList<>();+ private long nextMutationId = 1;++ synchronized Mutation stageParse(String statementName, CopyStatementParser.S3Statement statement) {+ Map<String, CopyStatementParser.S3Statement> statementsBefore = new LinkedHashMap<>(statements);+ Map<String, String> portalsBefore = new LinkedHashMap<>(portals);+ if (statement == null) {+ statements.remove(statementName);+ } else {+ statements.put(statementName, statement);+ }+ portals.entrySet().removeIf(entry -> entry.getValue().equals(statementName));+ return record(statementsBefore, portalsBefore);+ }++ synchronized Mutation stageBind(String portalName, String statementName) {+ Map<String, CopyStatementParser.S3Statement> statementsBefore = new LinkedHashMap<>(statements);+ Map<String, String> portalsBefore = new LinkedHashMap<>(portals);+ if (statements.containsKey(statementName)) {+ portals.put(portalName, statementName);+ } else {+ portals.remove(portalName);+ }+ return record(statementsBefore, portalsBefore);+ }++ synchronized Mutation stageClose(char targetType, String name) {+ if (targetType != 'S' && targetType != 'P') {+ throw new IllegalArgumentException("Close target type must be S or P");+ }++ Map<String, CopyStatementParser.S3Statement> statementsBefore = new LinkedHashMap<>(statements);+ Map<String, String> portalsBefore = new LinkedHashMap<>(portals);+ if (targetType == 'S') {+ statements.remove(name);+ portals.entrySet().removeIf(entry -> entry.getValue().equals(name));+ } else {+ portals.remove(name);+ }+ return record(statementsBefore, portalsBefore);+ }++ synchronized void confirm(Mutation mutation) {+ journal.removeIf(entry -> entry.mutation().equals(mutation));+ }++ synchronized void rejectFrom(Mutation mutation) {+ int rejectedIndex = -1;+ for (int i = 0; i < journal.size(); i++) {+ if (journal.get(i).mutation().equals(mutation)) {+ rejectedIndex = i;+ break;+ }+ }+ if (rejectedIndex < 0) {+ return;+ }++ // The caller rejects one pipelined mutation at a time, in journal order (see+ // BackendResponseCoordinator#discardQueuedOperationsBeforeSync), so this only ever needs to+ // undo the rejected mutation's own effect. A later mutation that built on it (e.g. a Bind+ // naming the statement this Parse defined) gets its own rejectFrom call once the coordinator+ // reaches it; blindly restoring the whole map back to this entry's "before" would also wipe+ // out any later, independent mutation that happened to run first (see+ // rejectingAnEarlierMutationRetainsLaterSyncCycleMutation). The entry's own "after" snapshot+ // is used rather than the live map or the next entry's "before": both go stale once an+ // earlier reject's cascading portal cleanup has already touched a key this mutation never+ // itself changed.+ JournalEntry rejected = journal.get(rejectedIndex);+ revertUnchangedSince(statements, rejected.statementsBefore(), rejected.statementsAfter());+ revertUnchangedSince(portals, rejected.portalsBefore(), rejected.portalsAfter());+ // A statement this mutation defined may just have been reverted away; drop any portal now+ // pointing at a statement that no longer exists, the same cleanup stageClose('S', ...) does.+ portals.entrySet().removeIf(entry -> !statements.containsKey(entry.getValue()));++ journal.remove(rejectedIndex);+ }++ /**+ * Reverts {@code live}'s entries for keys the rejected mutation changed (where {@code before}+ * and {@code after} differ), but only where the live map still holds the value the mutation set+ * ({@code after}): a key a later mutation has since changed again is left alone, since that+ * later mutation now owns it and will be reverted by its own {@link #rejectFrom} call if needed.+ */+ private static <K, V> void revertUnchangedSince(Map<K, V> live, Map<K, V> before, Map<K, V> after) {+ Set<K> touchedKeys = new LinkedHashSet<>(before.keySet());+ touchedKeys.addAll(after.keySet());+ for (K key : touchedKeys) {+ V beforeValue = before.get(key);+ V afterValue = after.get(key);+ if (Objects.equals(beforeValue, afterValue)) {+ continue;+ }+ if (!Objects.equals(live.get(key), afterValue)) {+ continue;+ }+ if (beforeValue == null) {+ live.remove(key);+ } else {+ live.put(key, beforeValue);+ }+ }+ }++ synchronized Optional<CopyStatementParser.S3Statement> statement(String statementName) {+ return Optional.ofNullable(statements.get(statementName));+ }++ synchronized Optional<CopyStatementParser.S3Statement> portal(String portalName) {+ String statementName = portals.get(portalName);+ return Optional.ofNullable(statementName).map(statements::get);+ }++ synchronized void clearPortals() {+ portals.clear();+ }++ synchronized void transactionEnded() {+ if (journal.isEmpty()) {+ portals.clear();+ return;+ }++ Map<String, String> expiredPortals = journal.get(0).portalsBefore();+ removeUnchangedExpiredPortals(portals, expiredPortals);+ for (int i = 0; i < journal.size(); i++) {+ JournalEntry entry = journal.get(i);+ Map<String, String> portalsBefore = new LinkedHashMap<>(entry.portalsBefore());+ removeUnchangedExpiredPortals(portalsBefore, expiredPortals);+ journal.set(i, new JournalEntry(entry.mutation(), entry.statementsBefore(), portalsBefore,+ entry.statementsAfter(), entry.portalsAfter()));+ }+ }++ synchronized void clear() {+ statements.clear();+ portals.clear();+ journal.clear();+ }++ /**+ * Journals a mutation with both its "before" snapshots (passed in, captured prior to the+ * caller's own action) and its "after" snapshots, captured here immediately once that action has+ * been applied. Storing "after" explicitly, rather than reconstructing it later from the next+ * entry's "before" or the live maps, keeps it a fixed historical fact: those two proxies go+ * stale once an earlier {@link #rejectFrom} has already run its own cascading portal cleanup.+ */+ private Mutation record(Map<String, CopyStatementParser.S3Statement> statementsBefore,+ Map<String, String> portalsBefore) {+ Mutation mutation = new Mutation(nextMutationId++);+ journal.add(new JournalEntry(+ mutation,+ statementsBefore,+ portalsBefore,+ new LinkedHashMap<>(statements),+ new LinkedHashMap<>(portals)));+ return mutation;+ }++ private static void removeUnchangedExpiredPortals(Map<String, String> candidates,+ Map<String, String> expiredPortals) {+ candidates.entrySet().removeIf(entry -> expiredPortals.containsKey(entry.getKey())+ && Objects.equals(expiredPortals.get(entry.getKey()), entry.getValue()));+ }++ record Mutation(long id) {+ }++ private record JournalEntry(+ Mutation mutation,+ Map<String, CopyStatementParser.S3Statement> statementsBefore,+ Map<String, String> portalsBefore,+ Map<String, CopyStatementParser.S3Statement> statementsAfter,+ Map<String, String> portalsAfter) {+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.javanew file mode 100644index 0000000000..09d67cf398--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java@@ -0,0 +1,275 @@+package io.github.hectorvent.floci.services.redshift.proxy;++import io.github.hectorvent.floci.services.s3.S3Service;+import java.io.IOException;+import java.io.OutputStream;+import java.net.Socket;++final class ExtendedS3Exchange {++ private record CopyResult(boolean succeeded, byte[] deferredReadyForQuery) {+ }++ private ExtendedS3Exchange() {+ }++ static void execute(Socket client, Socket backend,+ PostgresWireDecoder.FrontendMessage executeFrame,+ CopyStatementParser.S3Statement statement, S3Service s3Service,+ BackendResponseCoordinator coordinator, BackendResponseCoordinator.Ticket ticket) throws IOException {+ BackendResponseCoordinator.GateResult gate;+ try {+ gate = coordinator.awaitExtendedExecuteTurn(ticket);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IOException("Interrupted while awaiting the Extended Query Execute turn", e);+ }+ if (gate != BackendResponseCoordinator.GateResult.READY) {+ return;+ }++ boolean failed = true;+ byte[] deferredReadyForQuery = null;+ try {+ switch (statement) {+ case CopyStatementParser.S3CopyFrom copy -> {+ CopyResult result = runCopy(client, backend, executeFrame, copy, s3Service, coordinator);+ failed = !result.succeeded();+ deferredReadyForQuery = result.deferredReadyForQuery();+ }+ case CopyStatementParser.S3Unload unload -> {+ CopyResult result = runUnload(client, backend, executeFrame, unload, s3Service, coordinator);+ failed = !result.succeeded();+ deferredReadyForQuery = result.deferredReadyForQuery();+ }+ }+ } catch (IOException | RuntimeException e) {+ closeQuietly(client);+ closeQuietly(backend);+ throw e;+ } finally {+ coordinator.completeOwnedExecute(ticket, failed);+ }+ if (deferredReadyForQuery != null) {+ coordinator.onBackendFrame('Z', deferredReadyForQuery);+ }+ }++ private static CopyResult runCopy(Socket client, Socket backend,+ PostgresWireDecoder.FrontendMessage executeFrame,+ CopyStatementParser.S3CopyFrom spec, S3Service s3Service,+ BackendResponseCoordinator coordinator) throws IOException {+ OutputStream backendOut = backend.getOutputStream();+ backendOut.write(executeFrame.toPacketBytes());+ backendOut.flush();+ PostgresWireDecoder.FrontendMessage sync = readClientSync(client, backendOut);++ PostgresWireDecoder decoder = new PostgresWireDecoder(backend.getInputStream());+ PostgresWireDecoder.FrontendMessage first = nextOwnedFrame(client, decoder, coordinator);+ if (first.type() == 'E') {+ forward(client, first);+ forwardClientSyncToBackend(sync, backendOut, coordinator);+ return new CopyResult(false, null);+ }+ if (first.type() != 'G') {+ throw unexpected(first, "CopyInResponse");+ }++ S3CopySimulator.CopyInput input;+ try {+ input = S3CopySimulator.prepareCopy(spec, s3Service);+ } catch (S3CopySimulator.S3TransferException e) {+ S3CopySimulator.writeCopyFail(backendOut, e.getMessage());+ forwardClientSyncToBackend(sync, backendOut, coordinator);+ drainExecute(client, decoder, coordinator, false);+ sendError(client, e.sqlState(), e.getMessage());+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));+ }++ try {+ S3CopySimulator.streamCopyInput(input, backendOut);+ backendOut.write(new byte[]{'c', 0, 0, 0, 4});+ backendOut.flush();+ } catch (RuntimeException | IOException e) {+ S3CopySimulator.writeCopyFail(backendOut, e.getMessage());+ forwardClientSyncToBackend(sync, backendOut, coordinator);+ drainExecute(client, decoder, coordinator, false);+ String detail = e.getMessage() != null ? e.getMessage() : e.toString();+ sendError(client, "XX000", "S3 COPY failed: " + detail);+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));+ }++ forwardClientSyncToBackend(sync, backendOut, coordinator);+ PostgresWireDecoder.FrontendMessage terminal = drainExecute(client, decoder, coordinator, true);+ return new CopyResult(terminal.type() != 'E', null);+ }++ private static CopyResult runUnload(Socket client, Socket backend,+ PostgresWireDecoder.FrontendMessage executeFrame,+ CopyStatementParser.S3Unload spec, S3Service s3Service,+ BackendResponseCoordinator coordinator) throws IOException {+ OutputStream backendOut = backend.getOutputStream();+ backendOut.write(executeFrame.toPacketBytes());+ backendOut.flush();+ forwardClientSyncToBackend(client, backendOut, coordinator);++ PostgresWireDecoder decoder = new PostgresWireDecoder(backend.getInputStream());+ PostgresWireDecoder.FrontendMessage first = nextOwnedFrame(client, decoder, coordinator);+ if (first.type() == 'E') {+ forward(client, first);+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));+ }+ if (first.type() != 'H') {+ throw unexpected(first, "CopyOutResponse");+ }++ S3CopySimulator.UnloadCollector collector;+ try {+ collector = S3CopySimulator.prepareUnload(spec, s3Service);+ } catch (S3CopySimulator.S3TransferException e) {+ S3CopySimulator.writeCopyFail(backendOut, e.getMessage());+ drainExecute(client, decoder, coordinator, false);+ sendError(client, e.sqlState(), e.getMessage());+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));+ }++ try (collector) {+ while (true) {+ PostgresWireDecoder.FrontendMessage message = nextOwnedFrame(client, decoder, coordinator);+ if (message.type() == 'd') {+ try {+ collector.accept(message.body());+ } catch (S3CopySimulator.S3TransferException e) {+ collector.abort();+ S3CopySimulator.writeCopyFail(backendOut, e.getMessage());+ drainExecute(client, decoder, coordinator, false);+ sendError(client, e.sqlState(), e.getMessage());+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));+ }+ } else if (message.type() == 'c') {+ continue;+ } else if (message.type() == 'E') {+ collector.abort();+ forward(client, message);+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));+ } else if (message.type() == 'C') {+ try {+ collector.complete();+ } catch (S3CopySimulator.S3TransferException e) {+ S3CopySimulator.writeCopyFail(backendOut, e.getMessage());+ sendError(client, e.sqlState(), e.getMessage());+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));+ }+ forward(client, message);+ return new CopyResult(true, null);+ } else {+ throw unexpected(message, "CopyData, CopyDone, or CommandComplete");+ }+ }+ }+ }++ private static PostgresWireDecoder.FrontendMessage drainExecute(Socket client,+ PostgresWireDecoder decoder, BackendResponseCoordinator coordinator,+ boolean forwardTerminal) throws IOException {+ while (true) {+ PostgresWireDecoder.FrontendMessage message = nextOwnedFrame(client, decoder, coordinator);+ char type = message.type();+ if (type == 'C' || type == 'E' || type == 'I' || type == 's') {+ if (forwardTerminal) {+ forward(client, message);+ }+ return message;+ }+ if (type != 'd' && type != 'c') {+ throw unexpected(message, "CopyData, CopyDone, or an Execute terminal response");+ }+ }+ }++ private static byte[] drainReadyForQuery(Socket client, PostgresWireDecoder decoder,+ BackendResponseCoordinator coordinator) throws IOException {+ PostgresWireDecoder.FrontendMessage message = nextOwnedFrame(client, decoder, coordinator);+ if (message.type() != 'Z') {+ throw unexpected(message, "ReadyForQuery");+ }+ forward(client, message);+ return message.body();+ }++ private static PostgresWireDecoder.FrontendMessage nextOwnedFrame(Socket client,+ PostgresWireDecoder decoder, BackendResponseCoordinator coordinator) throws IOException {+ while (true) {+ PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();+ if (message == null) {+ throw new IOException("Backend closed during an Extended Query S3 exchange");+ }+ char type = message.type();+ if (type == 'N' || type == 'A' || type == 'S') {+ coordinator.onBackendFrame(type, message.body());+ forward(client, message);+ continue;+ }+ if (type == '1' || type == '2' || type == 'T' || type == 't' || type == 'n') {+ coordinator.onBackendFrame(type, message.body());+ forward(client, message);+ continue;+ }+ return message;+ }+ }++ private static PostgresWireDecoder.FrontendMessage readClientSync(Socket client, OutputStream backendOut)+ throws IOException {+ PostgresWireDecoder decoder = new PostgresWireDecoder(client.getInputStream());+ while (true) {+ PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();+ if (message == null) {+ throw new IOException("Client closed during an Extended Query S3 exchange");+ }+ if (message.type() == 'S') {+ return message;+ }+ if (message.type() != 'H') {+ throw new IOException("Expected Flush or Sync after an Extended Query S3 Execute");+ }+ backendOut.write(message.toPacketBytes());+ backendOut.flush();+ }+ }++ private static void forwardClientSyncToBackend(Socket client, OutputStream backendOut,+ BackendResponseCoordinator coordinator) throws IOException {+ forwardClientSyncToBackend(readClientSync(client, backendOut), backendOut, coordinator);+ }++ private static void forwardClientSyncToBackend(PostgresWireDecoder.FrontendMessage sync, OutputStream backendOut,+ BackendResponseCoordinator coordinator) throws IOException {+ coordinator.register(BackendResponseCoordinator.Operation.SYNC, null);+ backendOut.write(sync.toPacketBytes());+ backendOut.flush();+ }++ private static IOException unexpected(PostgresWireDecoder.FrontendMessage message, String expected) {+ return new IOException("Expected " + expected + " but backend sent " + message.type());+ }++ private static void sendError(Socket client, String sqlState, String message) throws IOException {+ forward(client, new PostgresWireDecoder.FrontendMessage(+ 'E', S3CopySimulator.errorBody(sqlState, message)));+ }++ private static void forward(Socket client, PostgresWireDecoder.FrontendMessage message) throws IOException {+ OutputStream out = client.getOutputStream();+ out.write(message.toPacketBytes());+ out.flush();+ }++ private static void closeQuietly(Socket socket) {+ try {+ socket.close();+ } catch (IOException ignored) {+ // The original protocol failure is more useful than a secondary socket-close failure.+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoder.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoder.javaindex b6bc43f358..b650f4a2ba 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoder.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoder.java@@ -1,11 +1,15 @@package io.github.hectorvent.floci.services.redshift.proxy;+import java.io.ByteArrayOutputStream;import java.io.EOFException;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.List;import java.util.Objects;+import java.util.function.IntPredicate;/*** Frames the PostgreSQL wire-protocol byte stream sent by the frontend (client).@@ -50,7 +54,7 @@ public class PostgresWireDecoder {* @throws IOException on an I/O error, a length below 4, or a length above {@link #MAX_MESSAGE_BYTES}.*/public FrontendMessage nextMessage() throws IOException {- return nextMessage(null);+ return nextMessage(null, ignored -> true);}/**@@ -66,6 +70,21 @@ public class PostgresWireDecoder {* @throws IOException on an I/O error, a length below 4, or a length above {@link #MAX_MESSAGE_BYTES}.*/public FrontendMessage nextMessage(OutputStream passthroughOut) throws IOException {+ return nextMessage(passthroughOut, type -> type == 'Q');+ }++ /**+ * Read the next frontend message, selecting which message types should be retained for+ * inspection. Unselected messages are streamed to {@code passthroughOut} when it is non-null.+ *+ * @param passthroughOut destination for opaque or oversized frames, or {@code null} to buffer+ * @param inspectType predicate identifying message types whose bodies should be retained+ * @return the message, or {@code null} on a clean end-of-stream between messages.+ * @throws EOFException if EOF is hit inside the length field or body.+ * @throws IOException on an I/O error, a length below 4, or a length above {@link #MAX_MESSAGE_BYTES}+ */+ public FrontendMessage nextMessage(OutputStream passthroughOut, IntPredicate inspectType) throws IOException {+ Objects.requireNonNull(inspectType, "inspectType must not be null");betweenMessages = true;int typeByte = in.read();@@ -85,7 +104,7 @@ public class PostgresWireDecoder {throw new IOException("Invalid message length: " + length);}- if (passthroughOut != null && (type != 'Q' || length > MAX_MESSAGE_BYTES)) {+ if (passthroughOut != null && (!inspectType.test(type) || length > MAX_MESSAGE_BYTES)) {passthroughOut.write(typeByte);passthroughOut.write(lengthBytes);int remaining = length - 4;@@ -117,6 +136,122 @@ public class PostgresWireDecoder {return new FrontendMessage(type, body);}+ /** Decode a Parse ({@code 'P'}) message body. */+ public ParseMessage decodeParse(FrontendMessage message) throws IOException {+ BodyCursor cursor = cursorFor(message, 'P');+ String statementName = cursor.readCString();+ String sql = cursor.readCString();+ int parameterCount = cursor.readInt16();+ List<Integer> parameterTypeOids = new ArrayList<>(parameterCount);+ for (int i = 0; i < parameterCount; i++) {+ parameterTypeOids.add(cursor.readInt32());+ }+ cursor.requireEnd();+ return new ParseMessage(statementName, sql, List.copyOf(parameterTypeOids));+ }++ /** Decode the leading names of a Bind ({@code 'B'}) message body. */+ public BindMessage decodeBind(FrontendMessage message) throws IOException {+ BodyCursor cursor = cursorFor(message, 'B');+ String portalName = cursor.readCString();+ String statementName = cursor.readCString();+ return new BindMessage(portalName, statementName);+ }++ /** Decode an Execute ({@code 'E'}) message body. */+ public ExecuteMessage decodeExecute(FrontendMessage message) throws IOException {+ BodyCursor cursor = cursorFor(message, 'E');+ String portalName = cursor.readCString();+ int maxRows = cursor.readInt32();+ cursor.requireEnd();+ return new ExecuteMessage(portalName, maxRows);+ }++ /** Decode a Describe ({@code 'D'}) message body. */+ public TargetMessage decodeDescribe(FrontendMessage message) throws IOException {+ return decodeTarget(message, 'D');+ }++ /** Decode a Close ({@code 'C'}) message body. */+ public TargetMessage decodeClose(FrontendMessage message) throws IOException {+ return decodeTarget(message, 'C');+ }++ /** Encode a rewritten Parse ({@code 'P'}) message, retaining its name and parameter OIDs. */+ public static byte[] encodeParse(ParseMessage parse, String sql) {+ Objects.requireNonNull(parse, "parse must not be null");+ String statementName = Objects.requireNonNull(parse.statementName(), "statementName must not be null");+ String rewrittenSql = Objects.requireNonNull(sql, "sql must not be null");+ List<Integer> parameterTypeOids = Objects.requireNonNull(+ parse.parameterTypeOids(), "parameterTypeOids must not be null");+ if (parameterTypeOids.size() > 0xFFFF) {+ throw new IllegalArgumentException("Too many PostgreSQL Parse parameter type OIDs: "+ + parameterTypeOids.size());+ }++ ByteArrayOutputStream body = new ByteArrayOutputStream();+ writeCString(body, statementName);+ writeCString(body, rewrittenSql);+ writeInt16(body, parameterTypeOids.size());+ for (Integer oid : parameterTypeOids) {+ writeInt32(body, Objects.requireNonNull(oid, "parameter type OID must not be null"));+ }++ byte[] bodyBytes = body.toByteArray();+ int length = bodyBytes.length + 4;+ byte[] packet = new byte[bodyBytes.length + 5];+ packet[0] = 'P';+ packet[1] = (byte) ((length >> 24) & 0xFF);+ packet[2] = (byte) ((length >> 16) & 0xFF);+ packet[3] = (byte) ((length >> 8) & 0xFF);+ packet[4] = (byte) (length & 0xFF);+ System.arraycopy(bodyBytes, 0, packet, 5, bodyBytes.length);+ return packet;+ }++ private TargetMessage decodeTarget(FrontendMessage message, char expectedType) throws IOException {+ BodyCursor cursor = cursorFor(message, expectedType);+ int targetByte = cursor.readByte();+ if (targetByte != 'S' && targetByte != 'P') {+ throw new IOException("Invalid PostgreSQL " + expectedType + " target type: " + targetByte);+ }+ String name = cursor.readCString();+ cursor.requireEnd();+ return new TargetMessage((char) targetByte, name);+ }++ private static BodyCursor cursorFor(FrontendMessage message, char expectedType) throws IOException {+ if (message == null) {+ throw new IOException("PostgreSQL message is missing");+ }+ if (message.type() != expectedType) {+ throw new IOException("Expected PostgreSQL message type " + expectedType+ + " but received " + message.type());+ }+ if (message.body() == null) {+ throw new IOException("PostgreSQL message body was streamed and is unavailable");+ }+ return new BodyCursor(message.body());+ }++ private static void writeCString(ByteArrayOutputStream out, String value) {+ byte[] bytes = value.getBytes(StandardCharsets.UTF_8);+ out.writeBytes(bytes);+ out.write(0);+ }++ private static void writeInt16(ByteArrayOutputStream out, int value) {+ out.write((value >> 8) & 0xFF);+ out.write(value & 0xFF);+ }++ private static void writeInt32(ByteArrayOutputStream out, int value) {+ out.write((value >> 24) & 0xFF);+ out.write((value >> 16) & 0xFF);+ out.write((value >> 8) & 0xFF);+ out.write(value & 0xFF);+ }+/** Read exactly {@code n} bytes or throw {@link EOFException}. */private byte[] readFully(int n) throws IOException {byte[] buf = new byte[n];@@ -131,6 +266,62 @@ public class PostgresWireDecoder {return buf;}+ private static final class BodyCursor {+ private final byte[] body;+ private int offset;++ private BodyCursor(byte[] body) {+ this.body = Objects.requireNonNull(body, "body must not be null");+ }++ private String readCString() throws IOException {+ int end = offset;+ while (end < body.length && body[end] != 0) {+ end++;+ }+ if (end == body.length) {+ throw new IOException("PostgreSQL message contains an unterminated string");+ }+ String value = new String(body, offset, end - offset, StandardCharsets.UTF_8);+ offset = end + 1;+ return value;+ }++ private int readByte() throws IOException {+ requireRemaining(1);+ return body[offset++] & 0xFF;+ }++ private int readInt16() throws IOException {+ requireRemaining(2);+ int value = ((body[offset] & 0xFF) << 8) | (body[offset + 1] & 0xFF);+ offset += 2;+ return value;+ }++ private int readInt32() throws IOException {+ requireRemaining(4);+ int value = ((body[offset] & 0xFF) << 24)+ | ((body[offset + 1] & 0xFF) << 16)+ | ((body[offset + 2] & 0xFF) << 8)+ | (body[offset + 3] & 0xFF);+ offset += 4;+ return value;+ }++ private void requireEnd() throws IOException {+ if (offset != body.length) {+ throw new IOException("PostgreSQL message has " + (body.length - offset) + " trailing bytes");+ }+ }++ private void requireRemaining(int count) throws IOException {+ if (body.length - offset < count) {+ throw new EOFException("PostgreSQL message body ended mid-field");+ }+ }+ }+/** Encode an SQL string as a Simple Query ({@code 'Q'}) packet with a NUL-terminated body. */public static byte[] encodeQuery(String sql) {if (sql == null) {@@ -189,4 +380,16 @@ public class PostgresWireDecoder {return packet;}}++ public record ParseMessage(String statementName, String sql, List<Integer> parameterTypeOids) {+ }++ public record BindMessage(String portalName, String statementName) {+ }++ public record ExecuteMessage(String portalName, int maxRows) {+ }++ public record TargetMessage(char targetType, String name) {+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.javaindex db9c9bcf5c..5b07d11637 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java@@ -9,25 +9,23 @@ import java.io.OutputStream;import java.net.Socket;import java.net.SocketTimeoutException;import java.util.concurrent.TimeUnit;-import java.util.concurrent.atomic.AtomicInteger;import java.util.concurrent.locks.LockSupport;import java.util.concurrent.locks.ReentrantLock;-import java.util.function.IntConsumer;/**- * Replaces the transparent {@code bridge()} for Redshift connections so Simple Query ({@code 'Q'})- * traffic can be inspected before it reaches the backing PostgreSQL container.+ * Replaces the transparent {@code bridge()} for Redshift connections so SQL in Simple and Extended+ * Query traffic can be inspected before it reaches the backing PostgreSQL container.** <p><b>Model.</b> Backend→client is a byte pump on a virtual thread. Client→backend is a- * framed loop: every frontend message is forwarded opaque except a {@code 'Q'}, whose SQL is run- * through {@link RedshiftSqlInterceptor#rewrite} and re-encoded only if it changed.+ * framed loop: SQL-bearing Query and Parse frames can be rewritten, while other frontend messages+ * are forwarded with their original bytes.** <p><b>Backend ownership.</b> A future interceptor that injects its own query and reads the reply* (the S3 COPY simulator) must own the backend exclusively and only start when no earlier request* is still being answered. {@link #backendLock} serialises pump reads against such an exchange, and- * {@link #outstandingResponses} plus {@link #pumpBetweenMessages} gate it to a safe boundary. If the- * backend cannot be caught idle within {@link #PUMP_PARK_WAIT_MS}, the exchange is skipped and the- * original {@code 'Q'} is forwarded (fail-open).+ * the response coordinator plus {@link #pumpBetweenMessages} gate it to a safe boundary. If the+ * backend cannot be caught idle within {@link #PUMP_PARK_WAIT_MS}, a Simple Query exchange is+ * skipped and the original {@code 'Q'} is forwarded (fail-open).*/public class RedshiftInterceptingBridge {@@ -43,13 +41,12 @@ public class RedshiftInterceptingBridge {private final Socket client;private final Socket backend;private final S3Service s3Service;+ private final ExtendedQuerySession session = new ExtendedQuerySession();+ private final BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);private final ReentrantLock backendLock = new ReentrantLock(true);private volatile boolean pumpBetweenMessages = true;- private final AtomicInteger outstandingResponses = new AtomicInteger(0);private volatile boolean pumpFinished = false;- /** Transaction-status byte from the backend's most recent {@code ReadyForQuery}; {@code 'I'} until one is seen. */- private volatile char lastBackendStatus = 'I';public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service) {this.client = client;@@ -76,7 +73,8 @@ public class RedshiftInterceptingBridge {while (true) {PostgresWireDecoder.FrontendMessage msg;try {- msg = decoder.nextMessage();+ msg = decoder.nextMessage(backendOut, type -> type == 'Q' || type == 'P'+ || type == 'B' || type == 'D' || type == 'E' || type == 'C' || type == 'S');} catch (SocketTimeoutException e) {if (decoder.isBetweenMessages()) {continue;@@ -88,66 +86,212 @@ public class RedshiftInterceptingBridge {break;}- if (!msg.isQuery()) {- if (msg.type() == 'S') {- outstandingResponses.incrementAndGet();- }- backendOut.write(msg.toPacketBytes());- backendOut.flush();- if (msg.type() == 'X') {+ if (msg.body() == null) {+ if (msg.type() == 'P') {+ session.clear();+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, null);+ } else if (msg.type() == 'B') {+ session.clearPortals();+ coordinator.register(BackendResponseCoordinator.Operation.BIND, null);+ } else if (msg.type() == 'D') {+ coordinator.register(BackendResponseCoordinator.Operation.DESCRIBE_PORTAL, null);+ } else if (msg.type() == 'E') {+ session.clearPortals();+ coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);+ } else if (msg.type() == 'C') {+ coordinator.register(BackendResponseCoordinator.Operation.CLOSE, null);+ } else if (msg.type() == 'X') {break;}continue;}-- String sql = msg.getSql();-- CopyStatementParser.S3Statement parsed = null;- try {- parsed = CopyStatementParser.parse(sql);- } catch (RuntimeException e) {- LOG.warnv("CopyStatementParser failed, forwarding original query: {0}", e.getMessage());- }-- if (parsed != null) {- CopyStatementParser.S3Statement statement = parsed;- boolean intercepted = runWithBackendOwned(() -> switch (statement) {- case CopyStatementParser.S3CopyFrom c ->- S3CopySimulator.runCopyFrom(client, backend, c, s3Service, lastBackendStatus,- status -> lastBackendStatus = (char) status);- case CopyStatementParser.S3Unload u ->- S3CopySimulator.runUnload(client, backend, u, s3Service, lastBackendStatus,- status -> lastBackendStatus = (char) status);- });- if (intercepted) {- continue;- }- }-- byte[] toBackend = msg.toPacketBytes();- try {- String rewritten = RedshiftSqlInterceptor.rewrite(sql);- if (rewritten != sql) {- toBackend = PostgresWireDecoder.encodeQuery(rewritten);+ switch (msg.type()) {+ case 'Q' -> handleSimpleQuery(msg, backendOut);+ case 'P' -> handleParse(decoder, msg, backendOut);+ case 'B' -> handleBind(decoder, msg, backendOut);+ case 'D' -> handleDescribe(decoder, msg, backendOut);+ case 'E' -> handleExecute(decoder, msg, backendOut);+ case 'C' -> handleClose(decoder, msg, backendOut);+ case 'S' -> {+ coordinator.register(BackendResponseCoordinator.Operation.SYNC, null);+ write(backendOut, msg.toPacketBytes());}- } catch (RuntimeException e) {- LOG.warnv("RedshiftSqlInterceptor failed, forwarding original query: {0}", e.getMessage());- toBackend = msg.toPacketBytes();+ default -> write(backendOut, msg.toPacketBytes());}- outstandingResponses.incrementAndGet();- backendOut.write(toBackend);- backendOut.flush();}} catch (IOException e) {LOG.debugv(e, "RedshiftInterceptingBridge client loop ended");} catch (Exception e) {LOG.warnv(e, "Unexpected error in RedshiftInterceptingBridge");} finally {+ coordinator.close();closeQuietly(client, "client");closeQuietly(backend, "backend");}}+ private void handleSimpleQuery(PostgresWireDecoder.FrontendMessage message, OutputStream backendOut)+ throws IOException {+ String sql = message.getSql();+ CopyStatementParser.S3Statement parsed = parseS3Statement(sql);+ if (parsed != null) {+ CopyStatementParser.S3Statement statement = parsed;+ boolean intercepted = runWithBackendOwned(() -> switch (statement) {+ case CopyStatementParser.S3CopyFrom copy -> S3CopySimulator.runCopyFrom(+ client, backend, copy, s3Service, coordinator.lastReadyStatus(),+ status -> coordinator.onBackendFrame('Z', new byte[]{(byte) status}));+ case CopyStatementParser.S3Unload unload -> S3CopySimulator.runUnload(+ client, backend, unload, s3Service, coordinator.lastReadyStatus(),+ status -> coordinator.onBackendFrame('Z', new byte[]{(byte) status}));+ });+ if (intercepted) {+ return;+ }+ }++ byte[] packet = message.toPacketBytes();+ try {+ String rewritten = RedshiftSqlInterceptor.rewrite(sql);+ if (!rewritten.equals(sql)) {+ packet = PostgresWireDecoder.encodeQuery(rewritten);+ }+ } catch (RuntimeException e) {+ LOG.warnv("RedshiftSqlInterceptor failed, forwarding original query: {0}", e.getMessage());+ }+ coordinator.register(BackendResponseCoordinator.Operation.SIMPLE_QUERY, null);+ write(backendOut, packet);+ }++ private void handleParse(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,+ OutputStream backendOut) throws IOException {+ PostgresWireDecoder.ParseMessage parse = decoder.decodeParse(message);+ CopyStatementParser.S3Statement statement = parse.parameterTypeOids().isEmpty()+ ? parseS3Statement(parse.sql()) : null;+ ExtendedQuerySession.Mutation mutation = session.stageParse(parse.statementName(), statement);+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, mutation);++ String rewritten = parse.sql();+ if (statement instanceof CopyStatementParser.S3CopyFrom copy) {+ rewritten = S3CopySimulator.copyBackendSql(copy);+ } else if (statement instanceof CopyStatementParser.S3Unload unload) {+ rewritten = S3CopySimulator.unloadBackendSql(unload);+ } else {+ try {+ rewritten = RedshiftSqlInterceptor.rewrite(parse.sql());+ } catch (RuntimeException e) {+ LOG.warnv("RedshiftSqlInterceptor failed, forwarding original Parse: {0}", e.getMessage());+ }+ }+ byte[] packet = rewritten.equals(parse.sql())+ ? message.toPacketBytes() : PostgresWireDecoder.encodeParse(parse, rewritten);+ write(backendOut, packet);+ }++ private void handleBind(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,+ OutputStream backendOut) throws IOException {+ PostgresWireDecoder.BindMessage bind = decoder.decodeBind(message);+ ExtendedQuerySession.Mutation mutation = session.stageBind(bind.portalName(), bind.statementName());+ coordinator.register(BackendResponseCoordinator.Operation.BIND, mutation);+ write(backendOut, message.toPacketBytes());+ }++ private void handleDescribe(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,+ OutputStream backendOut) throws IOException {+ PostgresWireDecoder.TargetMessage describe = decoder.decodeDescribe(message);+ BackendResponseCoordinator.Operation operation = describe.targetType() == 'S'+ ? BackendResponseCoordinator.Operation.DESCRIBE_STATEMENT+ : BackendResponseCoordinator.Operation.DESCRIBE_PORTAL;+ coordinator.register(operation, null);+ write(backendOut, message.toPacketBytes());+ }++ private void handleExecute(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,+ OutputStream backendOut) throws IOException {+ PostgresWireDecoder.ExecuteMessage execute = decoder.decodeExecute(message);+ BackendResponseCoordinator.Ticket ticket = coordinator.register(+ BackendResponseCoordinator.Operation.EXECUTE, null);+ CopyStatementParser.S3Statement statement = session.portal(execute.portalName()).orElse(null);+ if (statement == null) {+ write(backendOut, message.toPacketBytes());+ return;+ }+ runExtendedWithBackendOwned(message, statement, ticket);+ }++ private void handleClose(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,+ OutputStream backendOut) throws IOException {+ PostgresWireDecoder.TargetMessage close = decoder.decodeClose(message);+ ExtendedQuerySession.Mutation mutation = session.stageClose(close.targetType(), close.name());+ coordinator.register(BackendResponseCoordinator.Operation.CLOSE, mutation);+ write(backendOut, message.toPacketBytes());+ }++ private CopyStatementParser.S3Statement parseS3Statement(String sql) {+ try {+ return CopyStatementParser.parse(sql);+ } catch (RuntimeException e) {+ LOG.warnv("CopyStatementParser failed, forwarding original statement: {0}", e.getMessage());+ return null;+ }+ }++ private void runExtendedWithBackendOwned(PostgresWireDecoder.FrontendMessage executeFrame,+ CopyStatementParser.S3Statement statement, BackendResponseCoordinator.Ticket ticket) throws IOException {+ BackendResponseCoordinator.GateResult gate;+ try {+ gate = coordinator.awaitExtendedExecuteTurn(ticket);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IOException("Interrupted while awaiting Extended Query backend ownership", e);+ }+ if (gate != BackendResponseCoordinator.GateResult.READY) {+ return;+ }++ long deadlineNanos = System.nanoTime() + PUMP_PARK_WAIT_MS * 1_000_000L;+ boolean locked = false;+ while (!pumpFinished && System.nanoTime() < deadlineNanos) {+ try {+ long remainingMillis = Math.max(1L,+ TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()));+ locked = backendLock.tryLock(Math.min(PUMP_PARK_WAIT_MS, remainingMillis), TimeUnit.MILLISECONDS);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IOException("Interrupted while acquiring Extended Query backend ownership", e);+ }+ if (!locked) {+ continue;+ }+ if (pumpBetweenMessages) {+ break;+ }+ backendLock.unlock();+ locked = false;+ }+ if (!locked || !pumpBetweenMessages || pumpFinished) {+ if (locked) {+ backendLock.unlock();+ }+ throw new IOException("Could not take backend ownership for Extended Query Execute");+ }+ try {+ backend.setSoTimeout(EXCHANGE_READ_TIMEOUT_MS);+ ExtendedS3Exchange.execute(client, backend, executeFrame, statement, s3Service, coordinator, ticket);+ } finally {+ try {+ backend.setSoTimeout(PUMP_READ_TIMEOUT_MS);+ } catch (IOException e) {+ LOG.debugv(e, "could not restore backend read timeout (socket closing)");+ }+ backendLock.unlock();+ }+ }++ private static void write(OutputStream out, byte[] packet) throws IOException {+ out.write(packet);+ out.flush();+ }+/*** Take exclusive ownership of an idle backend at a wire-message boundary, run {@code exchange},* then release it. Returns {@code false} if the backend could not be caught idle within@@ -159,6 +303,18 @@ public class RedshiftInterceptingBridge {if (pumpFinished) {return false;}+ try {+ if (!coordinator.awaitIdle(PUMP_READ_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {+ if (System.nanoTime() >= deadlineNanos) {+ LOG.warn("backend did not go idle in time; forwarding the COPY unintercepted");+ return false;+ }+ continue;+ }+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IOException("interrupted while waiting for backend responses", e);+ }boolean locked;try {locked = backendLock.tryLock(PUMP_READ_TIMEOUT_MS, TimeUnit.MILLISECONDS);@@ -175,7 +331,7 @@ public class RedshiftInterceptingBridge {}boolean backendBusy;try {- backendBusy = !pumpBetweenMessages || outstandingResponses.get() > 0;+ backendBusy = !pumpBetweenMessages;if (!backendBusy || pumpFinished) {backend.setSoTimeout(EXCHANGE_READ_TIMEOUT_MS);try {@@ -200,10 +356,7 @@ public class RedshiftInterceptingBridge {}private void pumpBackendToClient() {- WireFrameTracker tracker = new WireFrameTracker(status -> {- lastBackendStatus = (char) status;- outstandingResponses.updateAndGet(v -> Math.max(0, v - 1));- });+ WireFrameTracker tracker = new WireFrameTracker(coordinator::onBackendFrame);try {InputStream backendIn = backend.getInputStream();OutputStream clientOut = client.getOutputStream();@@ -252,18 +405,20 @@ public class RedshiftInterceptingBridge {/*** Tracks PostgreSQL wire-message boundaries in a byte stream without buffering it. Each backend* message is {@code type(1) . length(int32, includes itself) . body}. {@link #betweenMessages()}- * is true exactly when the next byte would begin a new message; {@code onReadyForQuery} fires- * once per completed {@code 'Z'} with its one-byte transaction-status payload.+ * is true exactly when the next byte would begin a new message. The callback fires after every+ * complete frame and retains only the one-byte {@code ReadyForQuery} status payload.*/static final class WireFrameTracker {- private final IntConsumer onReadyForQuery;+ private final FrameCompletion onFrame;private int headerBytesSeen = 0;private final byte[] header = new byte[5];private long bodyRemaining = 0;+ private long declaredBodyLength = 0;private char pendingType = 0;+ private byte readyStatus;- WireFrameTracker(IntConsumer onReadyForQuery) {- this.onReadyForQuery = onReadyForQuery;+ WireFrameTracker(FrameCompletion onFrame) {+ this.onFrame = onFrame;}boolean betweenMessages() {@@ -274,9 +429,12 @@ public class RedshiftInterceptingBridge {for (int i = off; i < off + len; i++) {if (bodyRemaining > 0) {int current = buf[i] & 0xFF;+ if (pendingType == 'Z' && declaredBodyLength == 1) {+ readyStatus = (byte) current;+ }bodyRemaining--;- if (bodyRemaining == 0 && pendingType == 'Z') {- onReadyForQuery.accept(current);+ if (bodyRemaining == 0) {+ publishCompletion();}continue;}@@ -286,12 +444,24 @@ public class RedshiftInterceptingBridge {| ((header[3] & 0xFFL) << 8) | (header[4] & 0xFFL);pendingType = (char) (header[0] & 0xFF);bodyRemaining = Math.max(0, msgLen - 4);+ declaredBodyLength = bodyRemaining;headerBytesSeen = 0;- if (bodyRemaining == 0 && pendingType == 'Z') {- onReadyForQuery.accept('I');+ if (bodyRemaining == 0) {+ publishCompletion();}}}}++ private void publishCompletion() {+ byte[] body = pendingType == 'Z' && declaredBodyLength == 1+ ? new byte[]{readyStatus} : new byte[0];+ onFrame.accept(pendingType, body);+ }+ }++ @FunctionalInterface+ interface FrameCompletion {+ void accept(char type, byte[] body);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.javaindex a2145c3ccd..98624f8c1e 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java@@ -58,38 +58,51 @@ public final class S3CopySimulator {private S3CopySimulator() {}- /**- * @param txStatus the transaction-status byte from the client's last {@code ReadyForQuery}- * ({@code 'I'} idle, {@code 'T'} in a block, {@code 'E'} failed block); a- * synthesized error reports {@code 'I'} outside a block and {@code 'E'} inside.- * @return {@code true} when the exchange was handled (success or a clean error sent to the- * client). Part 4 always handles; the value exists so a later interceptor can decline.- */- public static boolean runCopyFrom(Socket client, Socket backend,- CopyStatementParser.S3CopyFrom spec, S3Service s3,- char txStatus) throws IOException {- return runCopyFrom(client, backend, spec, s3, txStatus, null);+ record CopyInput(CopyStatementParser.S3CopyFrom spec, List<String> keys, S3Service s3) {}- public static boolean runCopyFrom(Socket client, Socket backend,- CopyStatementParser.S3CopyFrom spec, S3Service s3,- char txStatus, IntConsumer onStatusChange) throws IOException {+ interface UnloadCollector extends AutoCloseable {+ void accept(byte[] body) throws IOException;++ void complete() throws IOException;++ void abort();++ @Override+ void close();+ }++ static final class S3TransferException extends RuntimeException {+ private final String sqlState;++ S3TransferException(String sqlState, String message, Throwable cause) {+ super(message, cause);+ this.sqlState = sqlState;+ }++ String sqlState() {+ return sqlState;+ }+ }++ static CopyInput prepareCopy(CopyStatementParser.S3CopyFrom spec, S3Service s3) {try {s3.authorizeAnonymousListBucket(spec.bucket());} catch (AwsException e) {- LOG.debugv(e, "ListBucket denied for COPY from s3://{0}/{1}", spec.bucket(), spec.keyOrPrefix());- sendAccessDenied(client, backend, spec, txStatus, onStatusChange);- return true;+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);}List<String> keys;try {keys = resolveKeys(spec, s3);} catch (AwsException e) {- LOG.warnv(e, "failed to list s3://{0}/{1} for COPY", spec.bucket(), spec.keyOrPrefix());- sendError(client, backend, SQLSTATE_INTERNAL,- "S3 COPY could not list s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), txStatus, onStatusChange);- return true;+ throw new S3TransferException(SQLSTATE_INTERNAL,+ "S3 COPY could not list s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);+ }+ if (keys.isEmpty()) {+ throw new S3TransferException(SQLSTATE_INTERNAL,+ "S3 object s3://" + spec.bucket() + "/" + spec.keyOrPrefix() + " not found", null);}try {@@ -97,19 +110,82 @@ public final class S3CopySimulator {s3.authorizeAnonymousGetObject(spec.bucket(), key);}} catch (AwsException e) {- LOG.debugv(e, "GetObject denied for COPY from s3://{0}/{1}", spec.bucket(), spec.keyOrPrefix());- sendAccessDenied(client, backend, spec, txStatus, onStatusChange);- return true;+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,+ "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);}+ return new CopyInput(spec, List.copyOf(keys), s3);+ }- if (keys.isEmpty()) {- sendError(client, backend, SQLSTATE_INTERNAL,- "S3 object s3://" + spec.bucket() + "/" + spec.keyOrPrefix() + " not found", txStatus, onStatusChange);+ static String copyBackendSql(CopyStatementParser.S3CopyFrom spec) {+ return fabricateCopy(spec);+ }++ static void streamCopyInput(CopyInput input, OutputStream backendOut) throws IOException {+ streamObjects(input.spec(), input.s3(), input.keys(), backendOut);+ }++ static UnloadCollector prepareUnload(CopyStatementParser.S3Unload spec, S3Service s3) {+ String probeKey = unloadDataKey(spec, 0);+ try {+ s3.authorizeAnonymousPutObject(spec.bucket(), probeKey);+ if (spec.manifest()) {+ s3.authorizeAnonymousPutObject(spec.bucket(), spec.prefix() + "manifest");+ }+ if (!spec.allowOverwrite()) {+ s3.authorizeAnonymousListBucket(spec.bucket());+ if (targetPrefixHasObjects(spec, s3)) {+ throw new S3TransferException(SQLSTATE_INTERNAL,+ "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()+ + " is not empty; specify ALLOWOVERWRITE to overwrite", null);+ }+ }+ } catch (AwsException e) {+ throw new S3TransferException(unloadWriteSqlState(e), unloadWriteMessage(e, spec), e);+ }++ if (!UNLOAD_HEAP_MIB.tryAcquire(UNLOAD_INITIAL_MIB)) {+ throw new S3TransferException(SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,+ "UNLOAD memory budget exhausted; retry shortly", null);+ }+ try {+ return new S3UnloadCollector(spec, s3);+ } catch (IOException e) {+ UNLOAD_HEAP_MIB.release(UNLOAD_INITIAL_MIB);+ throw new S3TransferException(SQLSTATE_INTERNAL, "UNLOAD failed: " + e.getMessage(), e);+ }+ }++ static String unloadBackendSql(CopyStatementParser.S3Unload spec) {+ return fabricateUnloadCopy(spec);+ }++ /**+ * @param txStatus the transaction-status byte from the client's last {@code ReadyForQuery}+ * ({@code 'I'} idle, {@code 'T'} in a block, {@code 'E'} failed block); a+ * synthesized error reports {@code 'I'} outside a block and {@code 'E'} inside.+ * @return {@code true} when the exchange was handled (success or a clean error sent to the+ * client). Part 4 always handles; the value exists so a later interceptor can decline.+ */+ public static boolean runCopyFrom(Socket client, Socket backend,+ CopyStatementParser.S3CopyFrom spec, S3Service s3,+ char txStatus) throws IOException {+ return runCopyFrom(client, backend, spec, s3, txStatus, null);+ }++ public static boolean runCopyFrom(Socket client, Socket backend,+ CopyStatementParser.S3CopyFrom spec, S3Service s3,+ char txStatus, IntConsumer onStatusChange) throws IOException {+ CopyInput input;+ try {+ input = prepareCopy(spec, s3);+ } catch (S3TransferException e) {+ LOG.debugv(e, "COPY preparation failed for s3://{0}/{1}", spec.bucket(), spec.keyOrPrefix());+ sendError(client, backend, e.sqlState(), e.getMessage(), txStatus, onStatusChange);return true;}OutputStream backendOut = backend.getOutputStream();- backendOut.write(PostgresWireDecoder.encodeQuery(fabricateCopy(spec)));+ backendOut.write(PostgresWireDecoder.encodeQuery(copyBackendSql(spec)));backendOut.flush();PostgresWireDecoder backendDecoder = new PostgresWireDecoder(backend.getInputStream());@@ -142,7 +218,7 @@ public final class S3CopySimulator {// a CopyFail to the backend, whose ErrorResponse/ReadyForQuery is relayed to the client;// or, if the backend is unreachable, one synthesized ErrorResponse/ReadyForQuery.try {- streamObjects(spec, s3, keys, backendOut);+ streamCopyInput(input, backendOut);writeCopyDone(backendOut);drainToReadyForQuery(backendDecoder, client, onStatusChange);} catch (RuntimeException | IOException e) {@@ -266,7 +342,7 @@ public final class S3CopySimulator {out.flush();}- private static void writeCopyFail(OutputStream out, String reason) throws IOException {+ static void writeCopyFail(OutputStream out, String reason) throws IOException {byte[] message = (reason == null ? "S3 COPY aborted" : reason).getBytes(StandardCharsets.UTF_8);out.write('f');out.write(intBytes(4 + message.length + 1));@@ -318,12 +394,6 @@ public final class S3CopySimulator {client.getOutputStream().flush();}- private static void sendAccessDenied(Socket client, Socket backend, CopyStatementParser.S3CopyFrom spec,- char txStatus, IntConsumer onStatusChange) throws IOException {- sendError(client, backend, SQLSTATE_INSUFFICIENT_PRIVILEGE,- "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), txStatus, onStatusChange);- }-private static void sendError(Socket client, Socket backend, String sqlState, String message,char txStatus, IntConsumer onStatusChange) throws IOException {if (txStatus == 'T' && backend != null && !backend.isClosed()) {@@ -399,7 +469,7 @@ public final class S3CopySimulator {}}- private static byte[] errorBody(String sqlState, String message) {+ static byte[] errorBody(String sqlState, String message) {ByteArrayOutputStream bytes = new ByteArrayOutputStream();writeField(bytes, 'S', "ERROR");writeField(bytes, 'C', sqlState);@@ -429,58 +499,29 @@ public final class S3CopySimulator {public static boolean runUnload(Socket client, Socket backend,CopyStatementParser.S3Unload spec, S3Service s3, char txStatus,IntConsumer onStatusChange) throws IOException {-- String probeKey = unloadDataKey(spec, 0);- String manifestKey = spec.prefix() + "manifest";+ UnloadCollector collector;try {- s3.authorizeAnonymousPutObject(spec.bucket(), probeKey);- if (spec.manifest()) {- s3.authorizeAnonymousPutObject(spec.bucket(), manifestKey);- }- } catch (AwsException e) {- LOG.debugv(e, "PutObject denied for UNLOAD to s3://{0}/{1}", spec.bucket(), spec.prefix());- sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);+ collector = prepareUnload(spec, s3);+ } catch (S3TransferException e) {+ LOG.debugv(e, "UNLOAD preparation failed for s3://{0}/{1}", spec.bucket(), spec.prefix());+ sendError(client, backend, e.sqlState(), e.getMessage(), txStatus, onStatusChange);return true;}- try {- if (!spec.allowOverwrite()) {- s3.authorizeAnonymousListBucket(spec.bucket());- if (targetPrefixHasObjects(spec, s3)) {- sendError(client, backend, SQLSTATE_INTERNAL,- "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()- + " is not empty; specify ALLOWOVERWRITE to overwrite", txStatus, onStatusChange);- return true;- }- }- } catch (AwsException e) {- LOG.debugv(e, "bucket check failed for UNLOAD to s3://{0}/{1}", spec.bucket(), spec.prefix());- sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);- return true;- }-- if (!UNLOAD_HEAP_MIB.tryAcquire(UNLOAD_INITIAL_MIB)) {- sendError(client, backend, SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,- "UNLOAD memory budget exhausted; retry shortly", txStatus, onStatusChange);- return true;- }- int[] heldMib = {UNLOAD_INITIAL_MIB};- try {- return runUnloadStreaming(client, backend, spec, s3, txStatus, onStatusChange, heldMib);- } finally {- UNLOAD_HEAP_MIB.release(heldMib[0]);+ try (collector) {+ return runUnloadStreaming(client, backend, spec, collector, txStatus, onStatusChange);}}private static boolean runUnloadStreaming(Socket client, Socket backend,- CopyStatementParser.S3Unload spec, S3Service s3, char txStatus,- IntConsumer onStatusChange, int[] heldMib) throws IOException {+ CopyStatementParser.S3Unload spec, UnloadCollector collector, char txStatus,+ IntConsumer onStatusChange) throws IOException {PostgresWireDecoder.FrontendMessage cmdComplete = null;PostgresWireDecoder.FrontendMessage readyForQuery = null;OutputStream backendOut = backend.getOutputStream();- backendOut.write(PostgresWireDecoder.encodeQuery(fabricateUnloadCopy(spec)));+ backendOut.write(PostgresWireDecoder.encodeQuery(unloadBackendSql(spec)));backendOut.flush();PostgresWireDecoder backendDecoder = new PostgresWireDecoder(backend.getInputStream());@@ -489,6 +530,7 @@ public final class S3CopySimulator {first = nextNonAsync(backendDecoder, client);} catch (IOException e) {LOG.warnv(e, "backend read failed while awaiting CopyOutResponse");+ collector.abort();closeQuietly(backend);sendError(client, null, SQLSTATE_INTERNAL,"UNLOAD failed: backend closed or timed out", txStatus, onStatusChange);@@ -496,6 +538,7 @@ public final class S3CopySimulator {return true;}if (first == null) {+ collector.abort();closeQuietly(backend);sendError(client, null, SQLSTATE_INTERNAL,"UNLOAD failed: backend closed before COPY started", txStatus, onStatusChange);@@ -503,217 +546,65 @@ public final class S3CopySimulator {return true;}if (first.type() != 'H') {- // Backend rejected the SELECT itself. Its ErrorResponse + ReadyForQuery are the client's one response.+ collector.abort();forward(client, first);drainToReadyForQuery(backendDecoder, client, onStatusChange);return true;}- long threshold = spec.maxFileSizeBytes() > 0 ? spec.maxFileSizeBytes() : UNLOAD_TARGET_FILE_BYTES;- String contentType = spec.gzip() ? "application/gzip" : "text/plain";- List<String> writtenKeys = new ArrayList<>();- List<Integer> writtenLengths = new ArrayList<>();-- ByteArrayOutputStream sink = new ByteArrayOutputStream();- OutputStream acc = spec.gzip() ? new GZIPOutputStream(sink) : sink;- long rawSlice = 0;- // Data-row bytes in the current slice, excluding a repeated header. Drives the size split so a- // header-only slice is never cut off on its own.- long slicePayload = 0;- long rawTotal = 0;- boolean lastByteNewline = true;- boolean warned = false;- int sliceIndex = 0;-- // With HEADER the backend emits the header row once, at the top of the stream. Capture it so- // it can be repeated at the start of every later slice, the way real Redshift writes one- // header per output file.- ByteArrayOutputStream headerBuf = spec.header() ? new ByteArrayOutputStream() : null;- byte[] headerBytes = null;- boolean capturingHeader = spec.header();-try {while (true) {- PostgresWireDecoder.FrontendMessage m = backendDecoder.nextMessage();- if (m == null) {- closeAcc(acc, sink);+ PostgresWireDecoder.FrontendMessage message = backendDecoder.nextMessage();+ if (message == null) {+ collector.abort();closeQuietly(backend);- if (spec.manifest()) {- deleteWritten(s3, spec, writtenKeys);- }sendError(client, null, SQLSTATE_INTERNAL,"UNLOAD failed: backend closed mid-stream", txStatus, onStatusChange);closeQuietly(client);return true;}- char t = m.type();- if (t == 'd') {- byte[] body = m.body();- int dataStart = 0;- if (capturingHeader) {- int nl = -1;- for (int i = 0; i < body.length; i++) {- if (body[i] == '\n') {- nl = i;- break;- }- }- int end = nl >= 0 ? nl + 1 : body.length;- headerBuf.write(body, 0, end);- if (nl >= 0) {- headerBytes = headerBuf.toByteArray();- headerBuf = null;- capturingHeader = false;- }- dataStart = end;- } else if (headerBytes != null && sliceIndex > 0 && rawSlice == 0) {- // First data of a later slice: repeat the captured header row.- acc.write(headerBytes);- rawSlice += headerBytes.length;- }- acc.write(body, 0, body.length);- rawSlice += body.length;- rawTotal += body.length;- slicePayload += body.length - dataStart;- if (body.length > 0) {- lastByteNewline = body[body.length - 1] == '\n';- }- if (!warned && rawTotal > UNLOAD_WARN_BYTES) {- warned = true;- LOG.warnv("UNLOAD result passed {0} bytes and is buffered a slice at a time "- + "(bucket={1}, prefix={2})", UNLOAD_WARN_BYTES, spec.bucket(), spec.prefix());- }- int wantMib = (int) ((rawSlice * 3) / (1024 * 1024)) + 1;- while (heldMib[0] < wantMib) {- if (!UNLOAD_HEAP_MIB.tryAcquire(1)) {- return abortUnload(client, backend, backendDecoder, acc, sink,- SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,- "UNLOAD memory budget exhausted; retry with a smaller result",- txStatus, onStatusChange);- }- heldMib[0]++;- }- if (rawTotal > UNLOAD_MAX_TOTAL_BYTES) {- closeAcc(acc, sink);- closeQuietly(backend);- if (spec.manifest()) {- deleteWritten(s3, spec, writtenKeys);- }- sendError(client, null,- SQLSTATE_PROGRAM_LIMIT_EXCEEDED,- "UNLOAD result exceeds the " + UNLOAD_MAX_TOTAL_BYTES + "-byte limit",- txStatus, onStatusChange);- closeQuietly(client);- return true;- }- if (slicePayload >= threshold && lastByteNewline) {- byte[] payload = finishSlice(acc, sink);- String key = unloadDataKey(spec, sliceIndex);- try {- s3.authorizeAnonymousPutObject(spec.bucket(), key);- s3.putObject(spec.bucket(), key, payload, contentType, Map.of());- } catch (AwsException e) {- LOG.debugv(e, "UNLOAD slice write to s3://{0}/{1} failed", spec.bucket(), key);- if (spec.manifest()) {- deleteWritten(s3, spec, writtenKeys);- }- return abortUnload(client, backend, backendDecoder, sink, sink,- unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);- }- writtenKeys.add(key);- writtenLengths.add(payload.length);- sliceIndex++;- int release = heldMib[0] - UNLOAD_INITIAL_MIB;- if (release > 0) {- UNLOAD_HEAP_MIB.release(release);- heldMib[0] = UNLOAD_INITIAL_MIB;- }- rawSlice = 0;- slicePayload = 0;- lastByteNewline = true;- sink = new ByteArrayOutputStream();- acc = spec.gzip() ? new GZIPOutputStream(sink) : sink;- }- } else if (t == 'c') {- // CopyDone, ignore- } else if (t == 'C') {- // CommandComplete, hold until we see 'Z'- cmdComplete = m;- } else if (t == 'Z') {- readyForQuery = m;+ char type = message.type();+ if (type == 'd') {+ collector.accept(message.body());+ } else if (type == 'C') {+ cmdComplete = message;+ } else if (type == 'Z') {+ readyForQuery = message;break;- } else if (t == 'E') {- closeAcc(acc, sink);- if (spec.manifest()) {- deleteWritten(s3, spec, writtenKeys);- }- forward(client, m);+ } else if (type == 'E') {+ collector.abort();+ forward(client, message);drainToReadyForQuery(backendDecoder, client, onStatusChange);return true;- } else if (t == 'N' || t == 'A' || t == 'S') {- forward(client, m);+ } else if (type == 'N' || type == 'A' || type == 'S') {+ forward(client, message);}}- } catch (AwsException e) {+ collector.complete();+ } catch (S3TransferException e) {LOG.debugv(e, "UNLOAD S3 operation failed during streaming");- if (spec.manifest()) {- deleteWritten(s3, spec, writtenKeys);+ if (readyForQuery != null) {+ sendError(client, backend, e.sqlState(), e.getMessage(), txStatus, onStatusChange);+ return true;+ }+ if (SQLSTATE_PROGRAM_LIMIT_EXCEEDED.equals(e.sqlState())) {+ closeQuietly(backend);+ sendError(client, null, e.sqlState(), e.getMessage(), txStatus, onStatusChange);+ closeQuietly(client);+ return true;}- return abortUnload(client, backend, backendDecoder, acc, sink,- unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);+ return abortUnload(client, backend, backendDecoder, collector,+ e.sqlState(), e.getMessage(), txStatus, onStatusChange);} catch (RuntimeException | IOException e) {LOG.warnv(e, "UNLOAD streaming failed");- if (spec.manifest()) {- deleteWritten(s3, spec, writtenKeys);- }String detail = e.getMessage() != null ? e.getMessage() : e.toString();- return abortUnload(client, backend, backendDecoder, acc, sink,- SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, txStatus, onStatusChange);- }-- // Flush the final slice. Write it when it carries rows, or when nothing has been written yet- // (a zero-row UNLOAD still emits one object, header only when HEADER was asked for). A result- // that ended exactly on a slice boundary leaves slicePayload == 0 with slices already written,- // so neither an empty nor a header-only trailer is produced.- byte[] payload = finishSlice(acc, sink);- if (slicePayload > 0 || writtenKeys.isEmpty()) {- String key = unloadDataKey(spec, sliceIndex);- try {- s3.authorizeAnonymousPutObject(spec.bucket(), key);- s3.putObject(spec.bucket(), key, payload, contentType, Map.of());- } catch (AwsException e) {- LOG.debugv(e, "UNLOAD final slice write to s3://{0}/{1} failed", spec.bucket(), key);- if (spec.manifest()) {- deleteWritten(s3, spec, writtenKeys);- }- sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec),- txStatus, onStatusChange);- return true;- } catch (RuntimeException e) {- LOG.warnv(e, "UNLOAD final slice write failed");- if (spec.manifest()) {- deleteWritten(s3, spec, writtenKeys);- }- String detail = e.getMessage() != null ? e.getMessage() : e.toString();- sendError(client, backend, SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, txStatus, onStatusChange);- return true;- }- writtenKeys.add(key);- writtenLengths.add(payload.length);- }-- if (spec.manifest()) {- try {- s3.authorizeAnonymousPutObject(spec.bucket(), spec.prefix() + "manifest");- s3.putObject(spec.bucket(), spec.prefix() + "manifest",- manifestJson(spec.bucket(), writtenKeys, writtenLengths).getBytes(StandardCharsets.UTF_8),- "application/json", Map.of());- } catch (RuntimeException e) {- LOG.warnv(e, "UNLOAD manifest write failed; removing partial data objects");- deleteWritten(s3, spec, writtenKeys);- sendError(client, backend, SQLSTATE_INTERNAL, "UNLOAD manifest write failed", txStatus, onStatusChange);+ if (readyForQuery != null) {+ sendError(client, backend, SQLSTATE_INTERNAL,+ "UNLOAD failed: " + detail, txStatus, onStatusChange);return true;}+ return abortUnload(client, backend, backendDecoder, collector,+ SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, txStatus, onStatusChange);}if (cmdComplete != null) {@@ -751,6 +642,196 @@ public final class S3CopySimulator {return sql.toString();}+ private static final class S3UnloadCollector implements UnloadCollector {+ private final CopyStatementParser.S3Unload spec;+ private final S3Service s3;+ private final long threshold;+ private final String contentType;+ private final List<String> writtenKeys = new ArrayList<>();+ private final List<Integer> writtenLengths = new ArrayList<>();++ private ByteArrayOutputStream sink = new ByteArrayOutputStream();+ private OutputStream acc;+ private ByteArrayOutputStream headerBuf;+ private byte[] headerBytes;+ private long rawSlice;+ private long slicePayload;+ private long rawTotal;+ private boolean lastByteNewline = true;+ private boolean warned;+ private boolean capturingHeader;+ private boolean finished;+ private boolean aborted;+ private boolean closed;+ private int sliceIndex;+ private int heldMib = UNLOAD_INITIAL_MIB;++ private S3UnloadCollector(CopyStatementParser.S3Unload spec, S3Service s3) throws IOException {+ this.spec = spec;+ this.s3 = s3;+ threshold = spec.maxFileSizeBytes() > 0 ? spec.maxFileSizeBytes() : UNLOAD_TARGET_FILE_BYTES;+ contentType = spec.gzip() ? "application/gzip" : "text/plain";+ acc = newAccumulator();+ headerBuf = spec.header() ? new ByteArrayOutputStream() : null;+ capturingHeader = spec.header();+ }++ @Override+ public void accept(byte[] body) throws IOException {+ ensureOpen();+ int dataStart = 0;+ if (capturingHeader) {+ int newline = -1;+ for (int i = 0; i < body.length; i++) {+ if (body[i] == '\n') {+ newline = i;+ break;+ }+ }+ int end = newline >= 0 ? newline + 1 : body.length;+ headerBuf.write(body, 0, end);+ if (newline >= 0) {+ headerBytes = headerBuf.toByteArray();+ headerBuf = null;+ capturingHeader = false;+ }+ dataStart = end;+ } else if (headerBytes != null && sliceIndex > 0 && rawSlice == 0) {+ acc.write(headerBytes);+ rawSlice += headerBytes.length;+ }++ acc.write(body, 0, body.length);+ rawSlice += body.length;+ rawTotal += body.length;+ slicePayload += body.length - dataStart;+ if (body.length > 0) {+ lastByteNewline = body[body.length - 1] == '\n';+ }+ if (!warned && rawTotal > UNLOAD_WARN_BYTES) {+ warned = true;+ LOG.warnv("UNLOAD result passed {0} bytes and is buffered a slice at a time "+ + "(bucket={1}, prefix={2})", UNLOAD_WARN_BYTES, spec.bucket(), spec.prefix());+ }+ acquireForCurrentSlice();+ if (rawTotal > UNLOAD_MAX_TOTAL_BYTES) {+ fail(SQLSTATE_PROGRAM_LIMIT_EXCEEDED,+ "UNLOAD result exceeds the " + UNLOAD_MAX_TOTAL_BYTES + "-byte limit", null);+ }+ if (slicePayload >= threshold && lastByteNewline) {+ writeCurrentSlice();+ resetSlice();+ }+ }++ @Override+ public void complete() throws IOException {+ ensureOpen();+ byte[] payload = finishSlice(acc, sink);+ if (slicePayload > 0 || writtenKeys.isEmpty()) {+ writePayload(payload, sliceIndex);+ }+ if (spec.manifest()) {+ try {+ String key = spec.prefix() + "manifest";+ s3.authorizeAnonymousPutObject(spec.bucket(), key);+ s3.putObject(spec.bucket(), key,+ manifestJson(spec.bucket(), writtenKeys, writtenLengths)+ .getBytes(StandardCharsets.UTF_8),+ "application/json", Map.of());+ } catch (RuntimeException e) {+ fail(SQLSTATE_INTERNAL, "UNLOAD manifest write failed", e);+ }+ }+ finished = true;+ }++ @Override+ public void abort() {+ if (aborted || finished) {+ return;+ }+ closeAcc(acc, sink);+ if (spec.manifest()) {+ deleteWritten(s3, spec, writtenKeys);+ }+ aborted = true;+ }++ @Override+ public void close() {+ if (closed) {+ return;+ }+ if (!finished && !aborted) {+ abort();+ }+ UNLOAD_HEAP_MIB.release(heldMib);+ heldMib = 0;+ closed = true;+ }++ private void acquireForCurrentSlice() {+ int wantedMib = (int) ((rawSlice * 3) / (1024 * 1024)) + 1;+ while (heldMib < wantedMib) {+ if (!UNLOAD_HEAP_MIB.tryAcquire(1)) {+ fail(SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,+ "UNLOAD memory budget exhausted; retry with a smaller result", null);+ }+ heldMib++;+ }+ }++ private void writeCurrentSlice() throws IOException {+ byte[] payload = finishSlice(acc, sink);+ writePayload(payload, sliceIndex);+ sliceIndex++;+ }++ private void writePayload(byte[] payload, int index) {+ String key = unloadDataKey(spec, index);+ try {+ s3.authorizeAnonymousPutObject(spec.bucket(), key);+ s3.putObject(spec.bucket(), key, payload, contentType, Map.of());+ } catch (AwsException e) {+ fail(unloadWriteSqlState(e), unloadWriteMessage(e, spec), e);+ } catch (RuntimeException e) {+ String detail = e.getMessage() != null ? e.getMessage() : e.toString();+ fail(SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, e);+ }+ writtenKeys.add(key);+ writtenLengths.add(payload.length);+ }++ private void resetSlice() throws IOException {+ int release = heldMib - UNLOAD_INITIAL_MIB;+ if (release > 0) {+ UNLOAD_HEAP_MIB.release(release);+ heldMib = UNLOAD_INITIAL_MIB;+ }+ rawSlice = 0;+ slicePayload = 0;+ lastByteNewline = true;+ sink = new ByteArrayOutputStream();+ acc = newAccumulator();+ }++ private OutputStream newAccumulator() throws IOException {+ return spec.gzip() ? new GZIPOutputStream(sink) : sink;+ }++ private void ensureOpen() {+ if (finished || aborted || closed) {+ throw new IllegalStateException("UNLOAD collector is no longer open");+ }+ }++ private void fail(String sqlState, String message, Throwable cause) {+ abort();+ throw new S3TransferException(sqlState, message, cause);+ }+ }+/** {@code 404} from the S3 layer means the bucket is absent, not that access was denied. */private static String unloadWriteSqlState(AwsException e) {return e.getHttpStatus() == 404 ? SQLSTATE_INTERNAL : SQLSTATE_INSUFFICIENT_PRIVILEGE;@@ -787,14 +868,12 @@ public final class S3CopySimulator {}private static boolean abortUnload(Socket client, Socket backend, PostgresWireDecoder backendDecoder,- OutputStream acc, ByteArrayOutputStream sink, String sqlState, String message,+ UnloadCollector collector, String sqlState, String message,char txStatus, IntConsumer onStatusChange) throws IOException {- closeAcc(acc, sink);+ collector.abort();try {drainBackendDiscarding(backendDecoder);} catch (IOException backendGone) {- // The backend is unreachable, so it cannot be resynchronized: close it and hand the- // client its one response synthesized, exactly as the backend-EOF branch does.LOG.debugv(backendGone, "backend unreachable while draining an aborted UNLOAD; synthesizing client error");closeQuietly(backend);sendError(client, null, sqlState, message, txStatus, onStatusChange);diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.javaindex 9fe447dd54..ba0db3196e 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java@@ -14,9 +14,11 @@ import org.junit.jupiter.api.Test;import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;+import java.io.InputStream;import java.nio.charset.StandardCharsets;import java.sql.Connection;import java.sql.DriverManager;+import java.sql.PreparedStatement;import java.sql.ResultSet;import java.sql.SQLException;import java.sql.Statement;@@ -31,6 +33,7 @@ import java.util.zip.GZIPOutputStream;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTest@@ -68,24 +71,100 @@ class RedshiftInterceptorIntegrationTest {}}- private static String jdbcUrl(Cluster c) {+ private static String jdbcUrl(Cluster cluster) {// Use 127.0.0.1 explicitly instead of c.getEndpoint().getAddress() to avoid UnknownHostException// in CI environments where floci.emulator.hostname is set to host.docker.internal.- // preferQueryMode=simple forces pgjdbc to use the simple query protocol ('Q' messages)- // rather than extended query protocol ('P'/'B'/'E'/'S' messages).- return "jdbc:postgresql://127.0.0.1:" + c.getEndpoint().getPort() + "/dev?preferQueryMode=simple";+ return "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort() + "/dev";+ }++ private static String namedPreparedJdbcUrl(Cluster cluster) {+ return jdbcUrl(cluster) + "?prepareThreshold=1";}private static Connection waitForConnection(Cluster cluster, String username, String password) throws SQLException {+ return waitForConnection(jdbcUrl(cluster), username, password);+ }++ private static Connection waitForConnection(String jdbcUrl, String username, String password) throws SQLException {try {return Awaitility.await().atMost(Duration.ofSeconds(30)).pollInterval(Duration.ofMillis(500)).ignoreExceptions()- .until(() -> DriverManager.getConnection(jdbcUrl(cluster), username, password), Objects::nonNull);+ .until(() -> DriverManager.getConnection(jdbcUrl, username, password), Objects::nonNull);} catch (ConditionTimeoutException e) {- return DriverManager.getConnection(jdbcUrl(cluster), username, password); // throw original+ return DriverManager.getConnection(jdbcUrl, username, password); // throw original+ }+ }++ @Test+ void preparedDdlCopyAndUnloadUseDefaultExtendedQuery() throws Exception {+ clusterId = "it-extended-prepared";+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");+ String bucket = "redshift-extended";+ s3.createBucket(bucket, "us-east-1");+ s3.putObject(bucket, "copy/data.txt",+ "1|alice\n2|bob\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());++ try (Connection connection = waitForConnection(cluster, "admin", "Secret123");+ PreparedStatement ddl = connection.prepareStatement(+ "CREATE TABLE ext_sales (id int ENCODE az64, name text) "+ + "DISTSTYLE KEY DISTKEY (id)");+ PreparedStatement copy = connection.prepareStatement(+ "COPY ext_sales FROM 's3://redshift-extended/copy/data.txt'");+ PreparedStatement unload = connection.prepareStatement(+ "UNLOAD ('select id, name from ext_sales order by id') "+ + "TO 's3://redshift-extended/unload/' ALLOWOVERWRITE")) {+ ddl.execute();+ copy.execute();+ unload.execute();+ try (ResultSet rows = connection.createStatement().executeQuery(+ "SELECT count(*) FROM ext_sales")) {+ assertTrue(rows.next());+ assertEquals(2, rows.getInt(1));+ }+ }++ List<S3Object> objects = s3.listObjects(bucket, "unload/", null, 100);+ assertEquals(1, objects.size());+ assertEquals("1|alice\n2|bob\n", new String(+ s3.getObject(bucket, objects.get(0).getKey()).getData(), StandardCharsets.UTF_8));+ }++ @Test+ void namedPreparedCopyAndUnloadCanBeExecutedTwice() throws Exception {+ clusterId = "it-extended-named";+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");+ String bucket = "redshift-extended-named";+ s3.createBucket(bucket, "us-east-1");+ s3.putObject(bucket, "copy/data.txt", "7|seven\n".getBytes(StandardCharsets.UTF_8),+ "text/plain", Map.of());++ try (Connection connection = waitForConnection(+ namedPreparedJdbcUrl(cluster), "admin", "Secret123")) {+ connection.createStatement().execute("CREATE TABLE named_sales (id int, name text)");+ try (PreparedStatement copy = connection.prepareStatement(+ "COPY named_sales FROM 's3://redshift-extended-named/copy/data.txt'")) {+ copy.execute();+ copy.execute();+ }+ try (PreparedStatement unload = connection.prepareStatement(+ "UNLOAD ('select id, name from named_sales order by id') "+ + "TO 's3://redshift-extended-named/unload/' ALLOWOVERWRITE")) {+ unload.execute();+ unload.execute();+ }+ try (ResultSet rows = connection.createStatement().executeQuery(+ "SELECT count(*) FROM named_sales")) {+ assertTrue(rows.next());+ assertEquals(2, rows.getInt(1));+ }}++ List<S3Object> objects = s3.listObjects(bucket, "unload/", null, 100);+ assertEquals(1, objects.size());+ assertEquals("7|seven\n7|seven\n", new String(+ s3.getObject(bucket, objects.get(0).getKey()).getData(), StandardCharsets.UTF_8));}@Test@@ -212,24 +291,26 @@ class RedshiftInterceptorIntegrationTest {s3.createBucket(bucket, "us-east-1");try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {- c.createStatement().execute("CREATE TABLE tx_test (id int)");- c.setAutoCommit(false);- Statement st = c.createStatement();- st.execute("INSERT INTO tx_test VALUES (1)");- assertThrows(- SQLException.class,- () -> st.execute("COPY tx_test FROM 's3://redshift-copy-it-tx/does/not/exist'"));- // The backend must now be in the aborted transaction state ('E').- // Subsequent statements in this transaction block must fail.- assertThrows(- SQLException.class,- () -> st.execute("INSERT INTO tx_test VALUES (2)"));- c.rollback();- c.setAutoCommit(true);- try (ResultSet rs = c.createStatement().executeQuery("SELECT count(*) FROM tx_test")) {- assertTrue(rs.next());- assertEquals(0, rs.getInt(1));- }+ assertTimeoutPreemptively(Duration.ofSeconds(15), () -> {+ c.createStatement().execute("CREATE TABLE tx_test (id int)");+ c.setAutoCommit(false);+ Statement st = c.createStatement();+ st.execute("INSERT INTO tx_test VALUES (1)");+ assertThrows(+ SQLException.class,+ () -> st.execute("COPY tx_test FROM 's3://redshift-copy-it-tx/does/not/exist'"));+ // The backend must now be in the aborted transaction state ('E').+ // Subsequent statements in this transaction block must fail.+ assertThrows(+ SQLException.class,+ () -> st.execute("INSERT INTO tx_test VALUES (2)"));+ c.rollback();+ c.setAutoCommit(true);+ try (ResultSet rs = c.createStatement().executeQuery("SELECT count(*) FROM tx_test")) {+ assertTrue(rs.next());+ assertEquals(0, rs.getInt(1));+ }+ });}}@@ -273,12 +354,12 @@ class RedshiftInterceptorIntegrationTest {c.createStatement().execute("UNLOAD ('select id from g_src order by id') TO 's3://redshift-unload-it-gzip/g/' GZIP");}- var objs = s3.listObjects(bucket, "g/", null, 100);+ List<S3Object> objs = s3.listObjects(bucket, "g/", null, 100);assertEquals(1, objs.size());assertTrue(objs.get(0).getKey().endsWith(".gz"), objs.get(0).getKey());byte[] data = s3.getObject(bucket, objs.get(0).getKey()).getData();ByteArrayOutputStream raw = new ByteArrayOutputStream();- try (var in = new GZIPInputStream(new ByteArrayInputStream(data))) {+ try (InputStream in = new GZIPInputStream(new ByteArrayInputStream(data))) {in.transferTo(raw);}assertEquals("7\n8\n", raw.toString(StandardCharsets.UTF_8));@@ -297,7 +378,7 @@ class RedshiftInterceptorIntegrationTest {c.createStatement().execute("UNLOAD ('select id from m_src') TO 's3://redshift-unload-it-manifest/m/' MANIFEST");}- var manifest = s3.getObject(bucket, "m/manifest");+ S3Object manifest = s3.getObject(bucket, "m/manifest");assertNotNull(manifest);String json = new String(manifest.getData(), StandardCharsets.UTF_8);assertTrue(json.contains("\"entries\""), json);diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.javanew file mode 100644index 0000000000..3ef223e0f2--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java@@ -0,0 +1,181 @@+package io.github.hectorvent.floci.services.redshift.proxy;++import org.junit.jupiter.api.Test;++import java.util.concurrent.CompletableFuture;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertTrue;++class BackendResponseCoordinatorTest {++ private static final byte[] EMPTY_BODY = new byte[0];++ @Test+ void parseBindAndDescribeCompleteBeforeExecuteBecomesReady() throws InterruptedException {+ ExtendedQuerySession session = new ExtendedQuerySession();+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");++ coordinator.register(BackendResponseCoordinator.Operation.PARSE, session.stageParse("s", copy));+ coordinator.register(BackendResponseCoordinator.Operation.BIND, session.stageBind("p", "s"));+ coordinator.register(BackendResponseCoordinator.Operation.DESCRIBE_PORTAL, null);+ BackendResponseCoordinator.Ticket execute = coordinator.register(+ BackendResponseCoordinator.Operation.EXECUTE, null);++ coordinator.onBackendFrame('1', EMPTY_BODY);+ coordinator.onBackendFrame('2', EMPTY_BODY);+ coordinator.onBackendFrame('n', EMPTY_BODY);++ assertEquals(BackendResponseCoordinator.GateResult.READY, coordinator.awaitTurn(execute));+ assertSame(copy, session.portal("p").orElseThrow());+ }++ @Test+ void statementDescribeWaitsPastParameterDescription() throws InterruptedException {+ ExtendedQuerySession session = new ExtendedQuerySession();+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);+ coordinator.register(BackendResponseCoordinator.Operation.DESCRIBE_STATEMENT, null);++ coordinator.onBackendFrame('t', EMPTY_BODY);++ assertFalse(coordinator.awaitIdle(1, TimeUnit.MILLISECONDS));+ coordinator.onBackendFrame('T', EMPTY_BODY);+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));+ }++ @Test+ void asynchronousFramesDoNotCompleteTheHeadOperation() throws InterruptedException {+ ExtendedQuerySession session = new ExtendedQuerySession();+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, null);++ coordinator.onBackendFrame('N', EMPTY_BODY);+ coordinator.onBackendFrame('A', EMPTY_BODY);+ coordinator.onBackendFrame('S', EMPTY_BODY);++ assertFalse(coordinator.awaitIdle(1, TimeUnit.MILLISECONDS));+ coordinator.onBackendFrame('1', EMPTY_BODY);+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));+ }++ @Test+ void extendedErrorRejectsPipelinedMutationsAndSkipsUntilSync() throws InterruptedException {+ ExtendedQuerySession session = new ExtendedQuerySession();+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, session.stageParse("s", copy));+ coordinator.register(BackendResponseCoordinator.Operation.BIND, session.stageBind("p", "s"));+ BackendResponseCoordinator.Ticket execute = coordinator.register(+ BackendResponseCoordinator.Operation.EXECUTE, null);++ coordinator.onBackendFrame('E', EMPTY_BODY);++ assertEquals(BackendResponseCoordinator.GateResult.SKIPPED_AFTER_ERROR,+ coordinator.awaitTurn(execute));+ assertTrue(session.statement("s").isEmpty());+ assertTrue(session.portal("p").isEmpty());+ assertTrue(coordinator.isDiscardingUntilSync());++ BackendResponseCoordinator.Ticket skipped = coordinator.register(+ BackendResponseCoordinator.Operation.EXECUTE, null);+ assertEquals(BackendResponseCoordinator.GateResult.SKIPPED_AFTER_ERROR,+ coordinator.awaitTurn(skipped));++ coordinator.register(BackendResponseCoordinator.Operation.SYNC, null);+ coordinator.onBackendFrame('Z', new byte[]{'I'});++ assertFalse(coordinator.isDiscardingUntilSync());+ assertEquals('I', coordinator.lastReadyStatus());+ }++ @Test+ void errorOnAnOperationWithoutAMutationRejectsLaterPipelinedMutations() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));+ coordinator.register(BackendResponseCoordinator.Operation.DESCRIBE_STATEMENT, null);+ coordinator.register(BackendResponseCoordinator.Operation.BIND, session.stageBind("p", "s"));++ coordinator.onBackendFrame('E', EMPTY_BODY);++ assertTrue(session.portal("p").isEmpty());+ }++ @Test+ void simpleQueryErrorCompletesOnlyWhenReadyForQueryArrives() throws InterruptedException {+ ExtendedQuerySession session = new ExtendedQuerySession();+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);+ coordinator.register(BackendResponseCoordinator.Operation.SIMPLE_QUERY, null);++ coordinator.onBackendFrame('E', EMPTY_BODY);++ assertFalse(coordinator.isDiscardingUntilSync());+ assertFalse(coordinator.awaitIdle(1, TimeUnit.MILLISECONDS));++ coordinator.onBackendFrame('Z', new byte[]{'I'});++ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));+ assertEquals('I', coordinator.lastReadyStatus());+ }++ @Test+ void readyForQueryInsideATransactionRetainsPortals() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));+ session.confirm(session.stageBind("p", "s"));+ coordinator.register(BackendResponseCoordinator.Operation.SYNC, null);++ coordinator.onBackendFrame('Z', new byte[]{'T'});++ assertSame(copy, session.portal("p").orElseThrow());+ assertEquals('T', coordinator.lastReadyStatus());+ }++ @Test+ void idleReadyForQueryKeepsPortalsStagedForTheNextPipelinedCycle() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));+ session.confirm(session.stageBind("old", "s"));+ coordinator.register(BackendResponseCoordinator.Operation.SYNC, null);+ coordinator.register(BackendResponseCoordinator.Operation.BIND, session.stageBind("next", "s"));++ coordinator.onBackendFrame('Z', new byte[]{'I'});++ assertTrue(session.portal("old").isEmpty());+ assertSame(copy, session.portal("next").orElseThrow());++ coordinator.onBackendFrame('E', EMPTY_BODY);+ assertTrue(session.portal("old").isEmpty());+ assertTrue(session.portal("next").isEmpty());+ }++ @Test+ void closeWakesABlockedWaiter() throws Exception {+ ExtendedQuerySession session = new ExtendedQuerySession();+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, null);+ BackendResponseCoordinator.Ticket execute = coordinator.register(+ BackendResponseCoordinator.Operation.EXECUTE, null);+ CompletableFuture<BackendResponseCoordinator.GateResult> result = CompletableFuture.supplyAsync(() -> {+ try {+ return coordinator.awaitTurn(execute);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ });++ coordinator.close();++ assertEquals(BackendResponseCoordinator.GateResult.CLOSED, result.get(1, TimeUnit.SECONDS));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySessionTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySessionTest.javanew file mode 100644index 0000000000..7c5c456077--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySessionTest.java@@ -0,0 +1,228 @@+package io.github.hectorvent.floci.services.redshift.proxy;++import org.junit.jupiter.api.Test;++import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++class ExtendedQuerySessionTest {++ @Test+ void bindMakesAnInterceptedStatementVisibleThroughItsPortal() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");++ ExtendedQuerySession.Mutation parse = session.stageParse("copy-s", copy);+ ExtendedQuerySession.Mutation bind = session.stageBind("copy-p", "copy-s");++ assertSame(copy, session.statement("copy-s").orElseThrow());+ assertSame(copy, session.portal("copy-p").orElseThrow());+ session.confirm(parse);+ session.confirm(bind);+ }++ @Test+ void ordinaryUnnamedParseAndBindReplaceInterceptedUnnamedMappings() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("", copy));+ session.confirm(session.stageBind("", ""));++ session.confirm(session.stageParse("", null));+ session.confirm(session.stageBind("", ""));++ assertTrue(session.statement("").isEmpty());+ assertTrue(session.portal("").isEmpty());+ }++ @Test+ void rejectingAParseRollsBackItAndEveryLaterPipelinedMutation() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ ExtendedQuerySession.Mutation parse = session.stageParse("s", copy);+ session.stageBind("p", "s");++ session.rejectFrom(parse);++ assertTrue(session.statement("s").isEmpty());+ assertTrue(session.portal("p").isEmpty());+ }++ @Test+ void namedStatementsCoexist() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement first = CopyStatementParser.parse("COPY first FROM 's3://b/one'");+ CopyStatementParser.S3Statement second = CopyStatementParser.parse("COPY second FROM 's3://b/two'");++ session.confirm(session.stageParse("first", first));+ session.confirm(session.stageParse("second", second));++ assertSame(first, session.statement("first").orElseThrow());+ assertSame(second, session.statement("second").orElseThrow());+ }++ @Test+ void unnamedPortalRebindingReplacesItsStatement() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement first = CopyStatementParser.parse("COPY first FROM 's3://b/one'");+ CopyStatementParser.S3Statement second = CopyStatementParser.parse("COPY second FROM 's3://b/two'");+ session.confirm(session.stageParse("first", first));+ session.confirm(session.stageParse("second", second));+ session.confirm(session.stageBind("", "first"));++ session.confirm(session.stageBind("", "second"));++ assertSame(second, session.portal("").orElseThrow());+ }++ @Test+ void twoPortalsCanReferenceOneStatement() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));++ session.confirm(session.stageBind("first", "s"));+ session.confirm(session.stageBind("second", "s"));++ assertSame(copy, session.portal("first").orElseThrow());+ assertSame(copy, session.portal("second").orElseThrow());+ }++ @Test+ void closingAStatementAlsoClosesItsPortals() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));+ session.confirm(session.stageBind("first", "s"));+ session.confirm(session.stageBind("second", "s"));++ session.confirm(session.stageClose('S', "s"));++ assertTrue(session.statement("s").isEmpty());+ assertTrue(session.portal("first").isEmpty());+ assertTrue(session.portal("second").isEmpty());+ }++ @Test+ void closingAPortalLeavesItsStatementAndOtherPortals() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));+ session.confirm(session.stageBind("first", "s"));+ session.confirm(session.stageBind("second", "s"));++ session.confirm(session.stageClose('P', "first"));++ assertSame(copy, session.statement("s").orElseThrow());+ assertTrue(session.portal("first").isEmpty());+ assertSame(copy, session.portal("second").orElseThrow());+ }++ @Test+ void rejectingAStatementCloseRestoresTheStatementAndItsPortals() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));+ session.confirm(session.stageBind("first", "s"));+ session.confirm(session.stageBind("second", "s"));++ ExtendedQuerySession.Mutation close = session.stageClose('S', "s");+ assertTrue(session.statement("s").isEmpty());+ assertTrue(session.portal("first").isEmpty());+ assertTrue(session.portal("second").isEmpty());++ session.rejectFrom(close);++ assertSame(copy, session.statement("s").orElseThrow());+ assertSame(copy, session.portal("first").orElseThrow());+ assertSame(copy, session.portal("second").orElseThrow());+ }++ @Test+ void rejectingAPortalCloseRestoresOnlyThatPortal() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));+ session.confirm(session.stageBind("p", "s"));++ ExtendedQuerySession.Mutation close = session.stageClose('P', "p");+ session.rejectFrom(close);++ assertSame(copy, session.statement("s").orElseThrow());+ assertSame(copy, session.portal("p").orElseThrow());+ }++ @Test+ void repeatedLookupDoesNotChangeRollbackState() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ ExtendedQuerySession.Mutation parse = session.stageParse("s", copy);++ assertSame(copy, session.statement("s").orElseThrow());+ assertSame(copy, session.statement("s").orElseThrow());+ session.rejectFrom(parse);++ assertTrue(session.statement("s").isEmpty());+ }++ @Test+ void rejectingAnEarlierMutationRetainsLaterSyncCycleMutation() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");++ ExtendedQuerySession.Mutation rejected = session.stageParse("invalid", null);+ ExtendedQuerySession.Mutation retained = session.stageParse("copy", copy);++ session.rejectFrom(rejected);++ assertSame(copy, session.statement("copy").orElseThrow());+ session.confirm(retained);+ }++ @Test+ void transactionEndClearsPortalsButRetainsStatements() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));+ session.confirm(session.stageBind("p", "s"));++ session.transactionEnded();++ assertSame(copy, session.statement("s").orElseThrow());+ assertTrue(session.portal("p").isEmpty());+ }++ @Test+ void clearPortalsRetainsStatements() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));+ session.confirm(session.stageBind("p", "s"));++ session.clearPortals();++ assertSame(copy, session.statement("s").orElseThrow());+ assertTrue(session.portal("p").isEmpty());+ }++ @Test+ void clearRemovesStatementsAndPortals() {+ ExtendedQuerySession session = new ExtendedQuerySession();+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");+ session.confirm(session.stageParse("s", copy));+ session.confirm(session.stageBind("p", "s"));++ session.clear();++ assertTrue(session.statement("s").isEmpty());+ assertTrue(session.portal("p").isEmpty());+ }++ @Test+ void closeRejectsAnInvalidTargetType() {+ ExtendedQuerySession session = new ExtendedQuerySession();++ assertThrows(IllegalArgumentException.class, () -> session.stageClose('X', "name"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.javanew file mode 100644index 0000000000..468ab11255--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java@@ -0,0 +1,222 @@+package io.github.hectorvent.floci.services.redshift.proxy;++import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.OutputStream;+import java.net.ServerSocket;+import java.net.Socket;+import java.net.SocketTimeoutException;+import java.nio.charset.StandardCharsets;+import java.util.List;+import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++class ExtendedS3ExchangeTest {++ private Socket simClient;+ private Socket simBackend;+ private Socket testClient;+ private Socket testBackend;+ private S3Service s3;+ private final AtomicReference<Throwable> backendFailure = new AtomicReference<>();++ @BeforeEach+ void setUp() throws IOException {+ ServerSocket clientListener = new ServerSocket(0);+ simClient = new Socket("localhost", clientListener.getLocalPort());+ testClient = clientListener.accept();+ clientListener.close();++ ServerSocket backendListener = new ServerSocket(0);+ simBackend = new Socket("localhost", backendListener.getLocalPort());+ testBackend = backendListener.accept();+ backendListener.close();+ s3 = mock(S3Service.class);+ }++ @AfterEach+ void tearDown() throws IOException {+ for (Socket socket : new Socket[]{simClient, simBackend, testClient, testBackend}) {+ if (socket != null && !socket.isClosed()) {+ socket.close();+ }+ }+ }++ @Test+ void copyInForwardsCommandCompleteWithoutReadyForQuery() throws Exception {+ when(s3.objectExists("b", "in/data")).thenReturn(true);+ when(s3.getObject("b", "in/data")).thenReturn(+ new S3Object("b", "in/data", "1|a\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));+ CopyStatementParser.S3CopyFrom copy = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "in/data", "|", 0, false, false, null);+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());+ BackendResponseCoordinator.Ticket ticket = coordinator.register(+ BackendResponseCoordinator.Operation.EXECUTE, null);+ ByteArrayOutputStream copied = new ByteArrayOutputStream();+ writeSync();+ Thread backend = backendThread(() -> playCopyIn(copied));++ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, coordinator, ticket);+ joinBackend(backend);++ PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClient.getInputStream());+ assertEquals('C', clientDecoder.nextMessage().type());+ assertEquals("1|a\n", copied.toString(StandardCharsets.US_ASCII));+ coordinator.onBackendFrame('Z', new byte[]{'I'});+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));+ testClient.setSoTimeout(200);+ assertThrows(SocketTimeoutException.class, () -> testClient.getInputStream().read());+ }++ @Test+ void copyOutWritesS3AndForwardsOnlyCommandComplete() throws Exception {+ Map<String, byte[]> written = new ConcurrentHashMap<>();+ when(s3.putObject(eq("b"), any(), any(), any(), any())).thenAnswer(invocation -> {+ written.put(invocation.getArgument(1), invocation.getArgument(2));+ return null;+ });+ CopyStatementParser.S3Unload unload = new CopyStatementParser.S3Unload(+ "select a from t", "b", "out/", "|", false, false, false,+ false, null, false, true, false, 0);+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());+ BackendResponseCoordinator.Ticket ticket = coordinator.register(+ BackendResponseCoordinator.Operation.EXECUTE, null);+ writeSync();+ Thread backend = backendThread(this::playCopyOut);++ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), unload, s3, coordinator, ticket);+ joinBackend(backend);++ assertEquals("1\n2\n", new String(written.get("out/000"), StandardCharsets.US_ASCII));+ PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClient.getInputStream());+ assertEquals('C', clientDecoder.nextMessage().type());+ coordinator.onBackendFrame('Z', new byte[]{'I'});+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));+ testClient.setSoTimeout(200);+ assertThrows(SocketTimeoutException.class, () -> testClient.getInputStream().read());+ }++ @Test+ void missingCopyInputSendsCopyFailAndWaitsForSyncRecovery() throws Exception {+ CopyStatementParser.S3CopyFrom copy = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "missing", "|", 0, false, false, null);+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());+ BackendResponseCoordinator.Ticket ticket = coordinator.register(+ BackendResponseCoordinator.Operation.EXECUTE, null);+ writeSync();+ Thread backend = backendThread(() -> {+ PostgresWireDecoder decoder = new PostgresWireDecoder(testBackend.getInputStream());+ assertEquals('E', decoder.nextMessage().type());+ testBackend.getOutputStream().write(new byte[]{'G', 0, 0, 0, 7, 0, 0, 0});+ testBackend.getOutputStream().flush();+ assertEquals('f', decoder.nextMessage().type());+ assertEquals('S', decoder.nextMessage().type());+ writeFrame(testBackend.getOutputStream(), 'E',+ "SERROR\0C57014\0MCOPY aborted\0\0".getBytes(StandardCharsets.US_ASCII));+ writeFrame(testBackend.getOutputStream(), 'Z', new byte[]{'E'});+ testBackend.getOutputStream().flush();+ });++ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, coordinator, ticket);+ joinBackend(backend);++ testClient.setSoTimeout(1_000);+ PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClient.getInputStream());+ PostgresWireDecoder.FrontendMessage error = clientDecoder.nextMessage();+ assertEquals('E', error.type());+ assertTrue(new String(error.body(), StandardCharsets.US_ASCII).contains("XX000"));+ assertTrue(new String(error.body(), StandardCharsets.US_ASCII).contains("s3://b/missing"));+ assertEquals('Z', clientDecoder.nextMessage().type());+ assertEquals('E', coordinator.lastReadyStatus());+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));+ }++ private static PostgresWireDecoder.FrontendMessage executeFrame() {+ return new PostgresWireDecoder.FrontendMessage('E', new byte[]{0, 0, 0, 0, 0});+ }++ private void playCopyIn(ByteArrayOutputStream copied) throws IOException {+ PostgresWireDecoder decoder = new PostgresWireDecoder(testBackend.getInputStream());+ assertEquals('E', decoder.nextMessage().type());+ testBackend.getOutputStream().write(new byte[]{'G', 0, 0, 0, 7, 0, 0, 0});+ testBackend.getOutputStream().flush();+ while (true) {+ PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();+ if (message.type() == 'd') {+ copied.write(message.body());+ } else if (message.type() == 'c') {+ break;+ }+ }+ assertEquals('S', decoder.nextMessage().type());+ writeCommandComplete();+ }++ private void playCopyOut() throws IOException {+ PostgresWireDecoder decoder = new PostgresWireDecoder(testBackend.getInputStream());+ assertEquals('E', decoder.nextMessage().type());+ OutputStream out = testBackend.getOutputStream();+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});+ writeFrame(out, 'd', "1\n".getBytes(StandardCharsets.US_ASCII));+ writeFrame(out, 'd', "2\n".getBytes(StandardCharsets.US_ASCII));+ out.write(new byte[]{'c', 0, 0, 0, 4});+ writeCommandComplete();+ assertEquals('S', decoder.nextMessage().type());+ }++ private void writeCommandComplete() throws IOException {+ writeFrame(testBackend.getOutputStream(), 'C', "COPY 2\0".getBytes(StandardCharsets.US_ASCII));+ testBackend.getOutputStream().flush();+ }++ private void writeSync() throws IOException {+ testClient.getOutputStream().write(new byte[]{'S', 0, 0, 0, 4});+ testClient.getOutputStream().flush();+ }++ private static void writeFrame(OutputStream out, char type, byte[] body) throws IOException {+ out.write(type);+ out.write(new byte[]{0, 0, 0, (byte) (4 + body.length)});+ out.write(body);+ }++ private Thread backendThread(BackendScript script) {+ return Thread.ofVirtual().start(() -> {+ try {+ script.run();+ } catch (Throwable failure) {+ backendFailure.compareAndSet(null, failure);+ }+ });+ }++ private void joinBackend(Thread backend) throws InterruptedException {+ backend.join();+ Throwable failure = backendFailure.get();+ if (failure != null) {+ throw new AssertionError("fake backend failed", failure);+ }+ }++ @FunctionalInterface+ private interface BackendScript {+ void run() throws IOException;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoderTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoderTest.javaindex 3ebd828056..15669188d0 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoderTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoderTest.java@@ -9,6 +9,7 @@ import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.nio.charset.StandardCharsets;+import java.util.List;import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -20,6 +21,96 @@ import static org.junit.jupiter.api.Assertions.assertTrue;class PostgresWireDecoderTest {+ @Test+ void decodesParseNameSqlAndParameterTypeOids() throws IOException {+ byte[] packet = frame('P', concat(+ cString("stmt-1"),+ cString("select $1::int"),+ int16(1),+ int32(23)));++ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(packet));+ PostgresWireDecoder.ParseMessage parse = decoder.decodeParse(decoder.nextMessage());++ assertEquals("stmt-1", parse.statementName());+ assertEquals("select $1::int", parse.sql());+ assertEquals(List.of(23), parse.parameterTypeOids());+ }++ @Test+ void decodesBindNamesWithoutRetainingValueSemantics() throws IOException {+ byte[] body = concat(cString("portal-1"), cString("stmt-1"), int16(0), int16(0), int16(0));+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(frame('B', body)));++ PostgresWireDecoder.BindMessage bind = decoder.decodeBind(decoder.nextMessage());++ assertEquals("portal-1", bind.portalName());+ assertEquals("stmt-1", bind.statementName());+ }++ @Test+ void decodesExecuteDescribeAndClose() throws IOException {+ PostgresWireDecoder.ExecuteMessage execute = decodeExecute(frame('E', concat(cString("p"), int32(0))));+ PostgresWireDecoder.TargetMessage describe = decodeDescribe(frame('D', concat(new byte[]{'S'}, cString("s"))));+ PostgresWireDecoder.TargetMessage close = decodeClose(frame('C', concat(new byte[]{'P'}, cString("p"))));++ assertEquals(new PostgresWireDecoder.ExecuteMessage("p", 0), execute);+ assertEquals(new PostgresWireDecoder.TargetMessage('S', "s"), describe);+ assertEquals(new PostgresWireDecoder.TargetMessage('P', "p"), close);+ }++ @Test+ void rejectsParseWithMissingCStringTerminator() {+ PostgresWireDecoder decoder = decoderFor('P', "stmt-1".getBytes(StandardCharsets.UTF_8));+ assertThrows(IOException.class, () -> decoder.decodeParse(decoder.nextMessage()));+ }++ @Test+ void rejectsParseWithTruncatedInt16() {+ byte[] body = concat(cString("stmt-1"), cString("select 1"), new byte[]{0});+ PostgresWireDecoder decoder = decoderFor('P', body);+ assertThrows(IOException.class, () -> decoder.decodeParse(decoder.nextMessage()));+ }++ @Test+ void rejectsParseWithTruncatedOidList() {+ byte[] body = concat(cString("stmt-1"), cString("select $1"), int16(1), new byte[]{0, 0});+ PostgresWireDecoder decoder = decoderFor('P', body);+ assertThrows(IOException.class, () -> decoder.decodeParse(decoder.nextMessage()));+ }++ @Test+ void rejectsDescribeWithInvalidTargetByte() {+ PostgresWireDecoder decoder = decoderFor('D', concat(new byte[]{'X'}, cString("name")));+ assertThrows(IOException.class, () -> decoder.decodeDescribe(decoder.nextMessage()));+ }++ @Test+ void rejectsCloseWithInvalidTargetByte() {+ PostgresWireDecoder decoder = decoderFor('C', concat(new byte[]{'X'}, cString("name")));+ assertThrows(IOException.class, () -> decoder.decodeClose(decoder.nextMessage()));+ }++ @Test+ void rejectsExecuteWithTrailingBytes() {+ PostgresWireDecoder decoder = decoderFor('E', concat(cString("p"), int32(0), new byte[]{1}));+ assertThrows(IOException.class, () -> decoder.decodeExecute(decoder.nextMessage()));+ }++ @Test+ void encodeParsePreservesNameAndTypeOidsWhileReplacingSql() throws IOException {+ PostgresWireDecoder.ParseMessage original = new PostgresWireDecoder.ParseMessage(+ "ddl", "CREATE TABLE t (id int ENCODE az64)", List.of(23, 25));++ byte[] encoded = PostgresWireDecoder.encodeParse(original, "CREATE TABLE t (id int)");+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(encoded));+ PostgresWireDecoder.ParseMessage decoded = decoder.decodeParse(decoder.nextMessage());++ assertEquals("ddl", decoded.statementName());+ assertEquals("CREATE TABLE t (id int)", decoded.sql());+ assertEquals(List.of(23, 25), decoded.parameterTypeOids());+ }+@Testvoid decodesASimpleQueryAndThenReportsEof() throws IOException {byte[] packet = PostgresWireDecoder.encodeQuery("SELECT 1");@@ -308,4 +399,69 @@ class PostgresWireDecoderTest {assertNull(msg.body());assertEquals(1 + (long) totalLength, bytesWritten[0]);}++ private static PostgresWireDecoder decoderFor(char type, byte[] body) {+ return new PostgresWireDecoder(new ByteArrayInputStream(frame(type, body)));+ }++ private static PostgresWireDecoder.ExecuteMessage decodeExecute(byte[] packet) throws IOException {+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(packet));+ return decoder.decodeExecute(decoder.nextMessage());+ }++ private static PostgresWireDecoder.TargetMessage decodeDescribe(byte[] packet) throws IOException {+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(packet));+ return decoder.decodeDescribe(decoder.nextMessage());+ }++ private static PostgresWireDecoder.TargetMessage decodeClose(byte[] packet) throws IOException {+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(packet));+ return decoder.decodeClose(decoder.nextMessage());+ }++ private static byte[] frame(char type, byte[] body) {+ int length = body.length + 4;+ byte[] packet = new byte[body.length + 5];+ packet[0] = (byte) type;+ packet[1] = (byte) ((length >> 24) & 0xFF);+ packet[2] = (byte) ((length >> 16) & 0xFF);+ packet[3] = (byte) ((length >> 8) & 0xFF);+ packet[4] = (byte) (length & 0xFF);+ System.arraycopy(body, 0, packet, 5, body.length);+ return packet;+ }++ private static byte[] cString(String value) {+ byte[] bytes = value.getBytes(StandardCharsets.UTF_8);+ byte[] result = new byte[bytes.length + 1];+ System.arraycopy(bytes, 0, result, 0, bytes.length);+ return result;+ }++ private static byte[] int16(int value) {+ return new byte[]{(byte) ((value >> 8) & 0xFF), (byte) (value & 0xFF)};+ }++ private static byte[] int32(int value) {+ return new byte[]{+ (byte) ((value >> 24) & 0xFF),+ (byte) ((value >> 16) & 0xFF),+ (byte) ((value >> 8) & 0xFF),+ (byte) (value & 0xFF)+ };+ }++ private static byte[] concat(byte[]... parts) {+ int length = 0;+ for (byte[] part : parts) {+ length += part.length;+ }+ byte[] result = new byte[length];+ int offset = 0;+ for (byte[] part : parts) {+ System.arraycopy(part, 0, result, offset, part.length);+ offset += part.length;+ }+ return result;+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.javaindex 2ab9e7b92e..2214d9476b 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java@@ -5,8 +5,10 @@ import io.github.hectorvent.floci.services.s3.model.S3Object;import org.jboss.logging.Logger;import org.junit.jupiter.api.AfterEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;import org.mockito.Mockito;+import java.io.ByteArrayOutputStream;import java.io.IOException;import java.io.OutputStream;import java.net.ServerSocket;@@ -123,6 +125,185 @@ class RedshiftInterceptingBridgeTest {assertArrayEquals(parsePacket, nextForwarded().toPacketBytes());}+ @Test+ void rewritesDdlParseWhilePreservingNameAndParameterOids() throws IOException {+ startBridge();+ PostgresWireDecoder.ParseMessage parse = new PostgresWireDecoder.ParseMessage(+ "ddl", "CREATE TABLE sales (id int ENCODE az64, d date) "+ + "DISTSTYLE KEY DISTKEY (id) SORTKEY (d)", List.of(23, 25));+ testClientEnd.getOutputStream().write(PostgresWireDecoder.encodeParse(parse, parse.sql()));+ testClientEnd.getOutputStream().flush();++ PostgresWireDecoder backendDecoder = new PostgresWireDecoder(testBackendEnd.getInputStream());+ PostgresWireDecoder.ParseMessage forwarded = backendDecoder.decodeParse(backendDecoder.nextMessage());++ assertEquals("ddl", forwarded.statementName());+ assertEquals(List.of(23, 25), forwarded.parameterTypeOids());+ String sql = forwarded.sql().toUpperCase();+ assertFalse(sql.contains("DISTSTYLE"), sql);+ assertFalse(sql.contains("DISTKEY"), sql);+ assertFalse(sql.contains("SORTKEY"), sql);+ assertFalse(sql.contains("ENCODE"), sql);+ }++ @Test+ void parameterizedCopyParseFallsOpenByteForByte() throws IOException {+ startBridge();+ PostgresWireDecoder.ParseMessage parse = new PostgresWireDecoder.ParseMessage(+ "copy", "COPY t FROM 's3://b/in/data'", List.of(25));+ byte[] packet = PostgresWireDecoder.encodeParse(parse, parse.sql());++ testClientEnd.getOutputStream().write(packet);+ testClientEnd.getOutputStream().flush();++ assertArrayEquals(packet, nextForwarded().toPacketBytes());+ }++ @Test+ void pipelinedExtendedCopyPreservesResponseOrderAndSyncReadyForQuery() throws Exception {+ startBridge();+ Mockito.when(s3Stub.objectExists("b", "in/data")).thenReturn(true);+ Mockito.when(s3Stub.getObject("b", "in/data")).thenReturn(+ new S3Object("b", "in/data", "1|a\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));+ AtomicReference<Throwable> backendFailure = new AtomicReference<>();+ Thread backend = Thread.ofVirtual().start(() -> {+ try {+ PostgresWireDecoder decoder = new PostgresWireDecoder(testBackendEnd.getInputStream());+ PostgresWireDecoder.ParseMessage parse = decoder.decodeParse(decoder.nextMessage());+ assertTrue(parse.sql().contains("FROM STDIN"), parse.sql());+ writeBackendFrame('1', new byte[0]);+ assertEquals('B', decoder.nextMessage().type());+ writeBackendFrame('2', new byte[0]);+ assertEquals('D', decoder.nextMessage().type());+ writeBackendFrame('n', new byte[0]);+ assertEquals('E', decoder.nextMessage().type());+ writeBackendFrame('G', new byte[]{0, 0, 0});+ while (decoder.nextMessage().type() != 'c') {+ // Consume CopyData until the bridge completes CopyIn.+ }+ assertEquals('S', decoder.nextMessage().type());+ writeBackendFrame('C', "COPY 1\0".getBytes(StandardCharsets.US_ASCII));+ writeBackendFrame('Z', new byte[]{'I'});+ } catch (Throwable failure) {+ backendFailure.compareAndSet(null, failure);+ }+ });++ PostgresWireDecoder.ParseMessage parse = new PostgresWireDecoder.ParseMessage(+ "copy-s", "COPY t FROM 's3://b/in/data'", List.of());+ ByteArrayOutputStream pipeline = new ByteArrayOutputStream();+ pipeline.write(PostgresWireDecoder.encodeParse(parse, parse.sql()));+ pipeline.write(frame('B', concat(cString("copy-p"), cString("copy-s"), new byte[6])));+ pipeline.write(frame('D', concat(new byte[]{'P'}, cString("copy-p"))));+ pipeline.write(frame('E', concat(cString("copy-p"), new byte[4])));+ pipeline.write(frame('S', new byte[0]));+ testClientEnd.getOutputStream().write(pipeline.toByteArray());+ testClientEnd.getOutputStream().flush();+ PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClientEnd.getInputStream());+ assertEquals('1', clientDecoder.nextMessage().type());+ assertEquals('2', clientDecoder.nextMessage().type());+ assertEquals('n', clientDecoder.nextMessage().type());+ assertEquals('C', clientDecoder.nextMessage().type());+ assertEquals('Z', clientDecoder.nextMessage().type());+ backend.join();+ if (backendFailure.get() != null) {+ throw new AssertionError("fake backend failed", backendFailure.get());+ }+ }++ @Test+ @Timeout(5)+ void extendedUnloadWithManifestCompletesAfterCopyOutCommandComplete() throws Exception {+ startBridge();+ Mockito.when(s3Stub.putObject(+ Mockito.eq("b"), Mockito.anyString(), Mockito.any(), Mockito.anyString(), Mockito.any()))+ .thenReturn(null);+ AtomicReference<Throwable> backendFailure = new AtomicReference<>();+ Thread backend = Thread.ofVirtual().start(() -> {+ try {+ PostgresWireDecoder decoder = new PostgresWireDecoder(testBackendEnd.getInputStream());+ PostgresWireDecoder.ParseMessage parse = decoder.decodeParse(decoder.nextMessage());+ assertTrue(parse.sql().startsWith("COPY (select id from t) TO STDOUT"), parse.sql());+ assertEquals('B', decoder.nextMessage().type());+ assertEquals('D', decoder.nextMessage().type());+ assertEquals('E', decoder.nextMessage().type());+ assertEquals('S', decoder.nextMessage().type());+ writeBackendFrame('1', new byte[0]);+ writeBackendFrame('2', new byte[0]);+ writeBackendFrame('n', new byte[0]);+ writeBackendFrame('H', new byte[]{0, 0, 0});+ writeBackendFrame('d', "1\n".getBytes(StandardCharsets.US_ASCII));+ writeBackendFrame('c', new byte[0]);+ writeBackendFrame('C', "COPY 1\0".getBytes(StandardCharsets.US_ASCII));+ writeBackendFrame('Z', new byte[]{'I'});+ } catch (Throwable failure) {+ backendFailure.compareAndSet(null, failure);+ }+ });++ PostgresWireDecoder.ParseMessage parse = new PostgresWireDecoder.ParseMessage(+ "unload-s", "UNLOAD ('select id from t') TO 's3://b/out/' MANIFEST", List.of());+ ByteArrayOutputStream pipeline = new ByteArrayOutputStream();+ pipeline.write(PostgresWireDecoder.encodeParse(parse, parse.sql()));+ pipeline.write(frame('B', concat(cString("unload-p"), cString("unload-s"), new byte[6])));+ pipeline.write(frame('D', concat(new byte[]{'P'}, cString("unload-p"))));+ pipeline.write(frame('E', concat(cString("unload-p"), new byte[4])));+ pipeline.write(frame('S', new byte[0]));+ testClientEnd.getOutputStream().write(pipeline.toByteArray());+ testClientEnd.getOutputStream().flush();+ testClientEnd.setSoTimeout(3000);++ backend.join(1000);+ if (backendFailure.get() != null) {+ throw new AssertionError("fake backend failed", backendFailure.get());+ }+ PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClientEnd.getInputStream());+ assertEquals('1', clientDecoder.nextMessage().type());+ assertEquals('2', clientDecoder.nextMessage().type());+ assertEquals('n', clientDecoder.nextMessage().type());+ assertEquals('C', clientDecoder.nextMessage().type());+ assertEquals('Z', clientDecoder.nextMessage().type());+ backend.join();+ if (backendFailure.get() != null) {+ throw new AssertionError("fake backend failed", backendFailure.get());+ }+ }++ private void writeBackendFrame(char type, byte[] body) throws IOException {+ testBackendEnd.getOutputStream().write(frame(type, body));+ testBackendEnd.getOutputStream().flush();+ }++ private static byte[] frame(char type, byte[] body) {+ int length = 4 + body.length;+ byte[] packet = new byte[5 + body.length];+ packet[0] = (byte) type;+ packet[1] = (byte) (length >>> 24);+ packet[2] = (byte) (length >>> 16);+ packet[3] = (byte) (length >>> 8);+ packet[4] = (byte) length;+ System.arraycopy(body, 0, packet, 5, body.length);+ return packet;+ }++ private static byte[] cString(String value) {+ return concat(value.getBytes(StandardCharsets.UTF_8), new byte[]{0});+ }++ private static byte[] concat(byte[]... values) {+ int length = 0;+ for (byte[] value : values) {+ length += value.length;+ }+ byte[] result = new byte[length];+ int offset = 0;+ for (byte[] value : values) {+ System.arraycopy(value, 0, result, offset, value.length);+ offset += value.length;+ }+ return result;+ }+@Testvoid pumpsBackendBytesToTheClientUnchanged() throws IOException {startBridge();diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.javaindex 9570a4158d..6b19bf101b 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java@@ -143,6 +143,58 @@ class S3CopySimulatorTest {"|", false, gzip, false, false, null, manifest, allowOverwrite, parallel, maxFileSize);}+ @Test+ void preparedCopyUsesSortedKeysAndBackendSql() {+ when(s3.objectExists("b", "prefix/")).thenReturn(false);+ when(s3.listObjectsWithPrefixes(+ eq("b"), eq("prefix/"), isNull(), anyInt(), isNull(), isNull()))+ .thenReturn(new S3Service.ListObjectsResult(+ List.of(+ new S3Object("b", "prefix/z", new byte[0], "text/plain"),+ new S3Object("b", "prefix/a", new byte[0], "text/plain")),+ List.of(), false, null));+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "prefix/", null, 0, false, true, null);++ S3CopySimulator.CopyInput input = S3CopySimulator.prepareCopy(spec, s3);++ assertEquals(List.of("prefix/a", "prefix/z"), input.keys());+ assertEquals("COPY t FROM STDIN WITH (FORMAT csv, DELIMITER ',')",+ S3CopySimulator.copyBackendSql(spec));+ }++ @Test+ void preparedCopyReportsMissingObject() {+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(+ "t", List.of(), "b", "missing", "|", 0, false, false, null);++ S3CopySimulator.S3TransferException error = assertThrows(+ S3CopySimulator.S3TransferException.class,+ () -> S3CopySimulator.prepareCopy(spec, s3));++ assertEquals("XX000", error.sqlState());+ assertEquals("S3 object s3://b/missing not found", error.getMessage());+ }++ @Test+ void unloadCollectorWritesOneEmptyObjectForZeroRows() throws Exception {+ Map<String, byte[]> written = new ConcurrentHashMap<>();+ when(s3.putObject(eq("b"), any(), any(), any(), any())).thenAnswer(invocation -> {+ written.put(invocation.getArgument(1), invocation.getArgument(2));+ return null;+ });+ CopyStatementParser.S3Unload spec = unloadSpec("b", "out/", false, false, true, false, 0);++ try (S3CopySimulator.UnloadCollector collector = S3CopySimulator.prepareUnload(spec, s3)) {+ collector.complete();+ }++ assertEquals(1, written.size());+ assertEquals(0, written.get("out/000").length);+ assertEquals("COPY (select a,b from t) TO STDOUT WITH (FORMAT text, DELIMITER '|')",+ S3CopySimulator.unloadBackendSql(spec));+ }+@Testvoid unloadWritesSingleObjectAndForwardsCommandComplete() throws Exception {Map<String, byte[]> written = new ConcurrentHashMap<>();