Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(redshift): intercept Extended Query protocol (#3474)

Vũ Thế Huye9454af85505
diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java
index a0da478d40..e5eab8f8b5 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftTest.java
@@ -6,6 +6,7 @@ import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.redshift.RedshiftClient;
import software.amazon.awssdk.services.redshift.model.Cluster;
import software.amazon.awssdk.services.redshift.model.CreateClusterRequest;
@@ -21,11 +22,14 @@ import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import java.net.ConnectException;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.Statement;
import java.time.Duration;
@@ -121,19 +125,33 @@ public class RedshiftTest {
.clusterIdentifier("test-cluster")
.build()).clusters().get(0);
String jdbcUrl = "jdbc:postgresql://" + cluster.endpoint().address() + ":"
- + cluster.endpoint().port() + "/dev?preferQueryMode=simple";
+ + cluster.endpoint().port() + "/dev";
S3Client s3 = TestFixtures.s3Client();
String bucket = "redshift-unload-compat";
s3.createBucket(CreateBucketRequest.builder().bucket(bucket).build());
+ s3.putObject(PutObjectRequest.builder().bucket(bucket).key("in/data.txt").contentType("text/plain").build(),
+ RequestBody.fromString("1|alice\n2|bob\n"));
try (Connection conn = DriverManager.getConnection(jdbcUrl, "admin", "Password123");
Statement st = conn.createStatement()) {
st.execute("DROP TABLE IF EXISTS unload_src");
- st.execute("CREATE TABLE unload_src (id int, name text)");
- st.execute("INSERT INTO unload_src VALUES (1, 'alice'), (2, 'bob')");
- st.execute("UNLOAD ('select id, name from unload_src order by id') "
- + "TO 's3://redshift-unload-compat/out/'");
+ try (PreparedStatement ddl = conn.prepareStatement("CREATE TABLE unload_src (id int, name text)")) {
+ ddl.execute();
+ }
+ try (PreparedStatement copy = conn.prepareStatement(
+ "COPY unload_src FROM 's3://redshift-unload-compat/in/data.txt'")) {
+ copy.execute();
+ }
+ try (PreparedStatement unload = conn.prepareStatement(
+ "UNLOAD ('select id, name from unload_src order by id') "
+ + "TO 's3://redshift-unload-compat/out/' ALLOWOVERWRITE")) {
+ unload.execute();
+ }
+ try (ResultSet rows = st.executeQuery("SELECT count(*) FROM unload_src")) {
+ assertTrue(rows.next());
+ assertEquals(2, rows.getInt(1));
+ }
}
ListObjectsV2Response listing = s3.listObjectsV2(ListObjectsV2Request.builder()
diff --git a/docs/services/redshift.md b/docs/services/redshift.md
index 32c24ca77f..4aec3461c1 100644
--- a/docs/services/redshift.md
+++ b/docs/services/redshift.md
@@ -141,7 +141,7 @@ print(cluster["Cluster"]["Endpoint"])
## SQL Interceptor
-Floci's Redshift auth proxy inspects frontend queries on the PostgreSQL wire protocol (Simple Query `'Q'` protocol) and rewrites common Redshift-specific table DDL so it runs on the plain PostgreSQL backend.
+Floci's Redshift auth proxy inspects frontend queries on the PostgreSQL wire protocol and rewrites common Redshift-specific table DDL so it runs on the plain PostgreSQL backend.
### DDL compatibility
@@ -170,12 +170,15 @@ order) through its own S3 service and streams the rows into the backing PostgreS
recognized: the statement is forwarded unchanged and PostgreSQL returns its own error.
- A multi-statement query whose COPY is followed by another statement is not intercepted; send the
COPY on its own.
-- Extended Query protocol COPY (a JDBC `PreparedStatement`, or pgjdbc's default
- `preferQueryMode=extended`) is not intercepted. Use `preferQueryMode=simple`.
+- Extended Query COPY is supported when the complete statement is present in `Parse` and has no
+ bind parameters. Zero-parameter JDBC `PreparedStatement` calls therefore work with pgjdbc's
+ default extended mode. Statements containing bind parameters are forwarded unchanged.
### Limitations
-- Emulation runs on the **Simple Query protocol** (`'Q'`) only. Extended Query protocol statements (`Parse`/`Bind`/`Execute`) pass through untouched, including anything a JDBC `PreparedStatement` sends, and, with the pgjdbc default `preferQueryMode=extended`, plain `Statement` calls too. Connect with `preferQueryMode=simple` to exercise the interceptor from JDBC.
+- DDL rewriting works in both Simple Query (`'Q'`) and Extended Query (`Parse`) flows. COPY and
+ UNLOAD interception in Extended Query is limited to zero-parameter statements fully present in
+ `Parse`; parameterized statements fail open to PostgreSQL.
- The rewrite is textual (regex-based). It masks single-quoted string literals first, so `DEFAULT` / `CHECK` string values are safe, but it is **not** comment-aware and does not recognize escape strings (`E'...'`): an apostrophe inside a `--` or `/* */` comment can make the rewrite skip a Redshift clause. That fails safe: the statement then reaches PostgreSQL, which returns its own syntax error, but avoid apostrophes-in-comments in `CREATE TABLE` / `ALTER TABLE`.
- A `rewrite` failure or any statement the interceptor does not recognize is forwarded unmodified (fail-open); PostgreSQL then rejects the Redshift-only syntax itself.
- Simple Query ('Q') messages larger than 16 MiB bypass the interceptor and stream through verbatim without heap buffering; non-query traffic also streams through with no size limit.
@@ -183,8 +186,8 @@ order) through its own S3 service and streams the rows into the backing PostgreS
### UNLOAD to S3
-`UNLOAD ('<select-statement>') TO 's3://<bucket>/<prefix>' [options]` sent over the
-Simple Query protocol runs the select on the backing PostgreSQL container and writes
+`UNLOAD ('<select-statement>') TO 's3://<bucket>/<prefix>' [options]` runs the select on the
+backing PostgreSQL container and writes
the result to S3 as one or more objects under `<prefix>`.
- Framing defaults to pipe-delimited text; `FORMAT CSV` (or `CSV`) switches to CSV
@@ -214,7 +217,8 @@ the result to S3 as one or more objects under `<prefix>`.
- Any other option (`PARQUET`, `ENCRYPTED`, `REGION`, `IAM_ROLE` / `CREDENTIALS`,
`ZSTD`, `EXTENSION`, `CLEANPATH`, `PARTITION`, and so on) is not intercepted; the
statement is forwarded and PostgreSQL reports its own error.
-- Extended Query protocol UNLOAD (a JDBC `PreparedStatement`) is not intercepted.
+- Extended Query UNLOAD is supported when the complete statement is present in `Parse` and has no
+ bind parameters. Parameterized statements are forwarded unchanged.
## Out of Scope
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java
new file mode 100644
index 0000000000..71ec88f443
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java
@@ -0,0 +1,295 @@
+package io.github.hectorvent.floci.services.redshift.proxy;
+
+import java.util.ArrayDeque;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+final class BackendResponseCoordinator {
+
+ private final ExtendedQuerySession session;
+ private final ReentrantLock lock = new ReentrantLock(true);
+ private final Condition changed = lock.newCondition();
+ private final ArrayDeque<PendingOperation> pending = new ArrayDeque<>();
+ private final Map<Long, GateResult> resolvedGates = new HashMap<>();
+ private long nextSequence = 1;
+ private boolean discardingUntilSync;
+ private boolean closed;
+ private char lastReadyStatus = 'I';
+
+ BackendResponseCoordinator(ExtendedQuerySession session) {
+ this.session = session;
+ }
+
+ Ticket register(Operation operation, ExtendedQuerySession.Mutation mutation) {
+ lock.lock();
+ try {
+ Ticket ticket = new Ticket(nextSequence++, operation);
+ if (discardingUntilSync && operation != Operation.SYNC) {
+ if (mutation != null) {
+ session.rejectFrom(mutation);
+ }
+ if (operation == Operation.EXECUTE) {
+ resolvedGates.put(ticket.sequence(), GateResult.SKIPPED_AFTER_ERROR);
+ }
+ } else {
+ pending.addLast(new PendingOperation(ticket, mutation));
+ }
+ changed.signalAll();
+ return ticket;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ GateResult awaitTurn(Ticket ticket) throws InterruptedException {
+ lock.lockInterruptibly();
+ try {
+ while (true) {
+ if (closed) {
+ return GateResult.CLOSED;
+ }
+ GateResult resolved = resolvedGates.remove(ticket.sequence());
+ if (resolved != null) {
+ return resolved;
+ }
+ PendingOperation head = pending.peekFirst();
+ if (!discardingUntilSync && head != null && head.ticket().equals(ticket)) {
+ return GateResult.READY;
+ }
+ changed.await();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ GateResult awaitExtendedExecuteTurn(Ticket ticket) throws InterruptedException {
+ lock.lockInterruptibly();
+ try {
+ while (true) {
+ if (closed) {
+ return GateResult.CLOSED;
+ }
+ GateResult resolved = resolvedGates.remove(ticket.sequence());
+ if (resolved != null) {
+ return resolved;
+ }
+ if (!discardingUntilSync && canOwnExtendedExecute(ticket)) {
+ return GateResult.READY;
+ }
+ changed.await();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ boolean awaitIdle(long timeout, TimeUnit unit) throws InterruptedException {
+ long remaining = unit.toNanos(timeout);
+ lock.lockInterruptibly();
+ try {
+ while ((!pending.isEmpty() || discardingUntilSync) && !closed) {
+ if (remaining <= 0) {
+ return false;
+ }
+ remaining = changed.awaitNanos(remaining);
+ }
+ return !closed && pending.isEmpty() && !discardingUntilSync;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ void onBackendFrame(char type, byte[] body) {
+ lock.lock();
+ try {
+ if (type == 'Z') {
+ updateReadyStatus(body);
+ }
+ if (isAsynchronous(type)) {
+ return;
+ }
+
+ PendingOperation head = pending.peekFirst();
+ if (head == null) {
+ changed.signalAll();
+ return;
+ }
+
+ Operation operation = head.ticket().operation();
+ if (type == 'E') {
+ if (operation != Operation.SIMPLE_QUERY) {
+ pending.removeFirst();
+ reject(head);
+ resolveSkippedGate(head);
+ discardingUntilSync = true;
+ discardQueuedOperationsBeforeSync();
+ }
+ changed.signalAll();
+ return;
+ }
+
+ if (completes(operation, type)) {
+ pending.removeFirst();
+ confirm(head);
+ if (operation == Operation.SYNC) {
+ discardingUntilSync = false;
+ }
+ changed.signalAll();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ void completeOwnedExecute(Ticket ticket, boolean failed) {
+ lock.lock();
+ try {
+ PendingOperation owned = pending.stream()
+ .filter(operation -> operation.ticket().equals(ticket))
+ .findFirst()
+ .orElse(null);
+ if (owned == null) {
+ return;
+ }
+ if (ticket.operation() != Operation.EXECUTE) {
+ throw new IllegalStateException("Only an Execute ticket can be completed as backend-owned");
+ }
+ pending.remove(owned);
+ confirm(owned);
+ if (failed) {
+ discardingUntilSync = true;
+ discardQueuedOperationsBeforeSync();
+ }
+ changed.signalAll();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ boolean isDiscardingUntilSync() {
+ lock.lock();
+ try {
+ return discardingUntilSync;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ char lastReadyStatus() {
+ lock.lock();
+ try {
+ return lastReadyStatus;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ void close() {
+ lock.lock();
+ try {
+ closed = true;
+ pending.clear();
+ resolvedGates.clear();
+ session.clear();
+ changed.signalAll();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private void discardQueuedOperationsBeforeSync() {
+ while (!pending.isEmpty() && pending.peekFirst().ticket().operation() != Operation.SYNC) {
+ PendingOperation skipped = pending.removeFirst();
+ reject(skipped);
+ resolveSkippedGate(skipped);
+ }
+ }
+
+ private void resolveSkippedGate(PendingOperation operation) {
+ if (operation.ticket().operation() == Operation.EXECUTE) {
+ resolvedGates.put(operation.ticket().sequence(), GateResult.SKIPPED_AFTER_ERROR);
+ }
+ }
+
+ private void updateReadyStatus(byte[] body) {
+ if (body != null && body.length == 1) {
+ lastReadyStatus = (char) body[0];
+ if (lastReadyStatus == 'I') {
+ session.transactionEnded();
+ }
+ }
+ }
+
+ private void confirm(PendingOperation operation) {
+ if (operation.mutation() != null) {
+ session.confirm(operation.mutation());
+ }
+ }
+
+ private void reject(PendingOperation operation) {
+ if (operation.mutation() != null) {
+ session.rejectFrom(operation.mutation());
+ }
+ }
+
+ private boolean canOwnExtendedExecute(Ticket ticket) {
+ for (PendingOperation operation : pending) {
+ if (operation.ticket().equals(ticket)) {
+ return true;
+ }
+ if (!isExtendedExecutePreamble(operation.ticket().operation())) {
+ return false;
+ }
+ }
+ return false;
+ }
+
+ private static boolean isExtendedExecutePreamble(Operation operation) {
+ return operation == Operation.PARSE
+ || operation == Operation.BIND
+ || operation == Operation.DESCRIBE_STATEMENT
+ || operation == Operation.DESCRIBE_PORTAL;
+ }
+
+ private static boolean isAsynchronous(char type) {
+ return type == 'N' || type == 'A' || type == 'S';
+ }
+
+ private static boolean completes(Operation operation, char responseType) {
+ return switch (operation) {
+ case SIMPLE_QUERY, SYNC -> responseType == 'Z';
+ case PARSE -> responseType == '1';
+ case BIND -> responseType == '2';
+ case DESCRIBE_STATEMENT, DESCRIBE_PORTAL -> responseType == 'T' || responseType == 'n';
+ case CLOSE -> responseType == '3';
+ case EXECUTE -> responseType == 'C' || responseType == 'I' || responseType == 's';
+ };
+ }
+
+ enum Operation {
+ SIMPLE_QUERY,
+ PARSE,
+ BIND,
+ DESCRIBE_STATEMENT,
+ DESCRIBE_PORTAL,
+ CLOSE,
+ EXECUTE,
+ SYNC
+ }
+
+ enum GateResult {
+ READY,
+ SKIPPED_AFTER_ERROR,
+ CLOSED
+ }
+
+ record Ticket(long sequence, Operation operation) {
+ }
+
+ private record PendingOperation(Ticket ticket, ExtendedQuerySession.Mutation mutation) {
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySession.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySession.java
new file mode 100644
index 0000000000..1e03c751af
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySession.java
@@ -0,0 +1,191 @@
+package io.github.hectorvent.floci.services.redshift.proxy;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+final class ExtendedQuerySession {
+
+ private final Map<String, CopyStatementParser.S3Statement> statements = new LinkedHashMap<>();
+ private final Map<String, String> portals = new LinkedHashMap<>();
+ private final List<JournalEntry> journal = new ArrayList<>();
+ private long nextMutationId = 1;
+
+ synchronized Mutation stageParse(String statementName, CopyStatementParser.S3Statement statement) {
+ Map<String, CopyStatementParser.S3Statement> statementsBefore = new LinkedHashMap<>(statements);
+ Map<String, String> portalsBefore = new LinkedHashMap<>(portals);
+ if (statement == null) {
+ statements.remove(statementName);
+ } else {
+ statements.put(statementName, statement);
+ }
+ portals.entrySet().removeIf(entry -> entry.getValue().equals(statementName));
+ return record(statementsBefore, portalsBefore);
+ }
+
+ synchronized Mutation stageBind(String portalName, String statementName) {
+ Map<String, CopyStatementParser.S3Statement> statementsBefore = new LinkedHashMap<>(statements);
+ Map<String, String> portalsBefore = new LinkedHashMap<>(portals);
+ if (statements.containsKey(statementName)) {
+ portals.put(portalName, statementName);
+ } else {
+ portals.remove(portalName);
+ }
+ return record(statementsBefore, portalsBefore);
+ }
+
+ synchronized Mutation stageClose(char targetType, String name) {
+ if (targetType != 'S' && targetType != 'P') {
+ throw new IllegalArgumentException("Close target type must be S or P");
+ }
+
+ Map<String, CopyStatementParser.S3Statement> statementsBefore = new LinkedHashMap<>(statements);
+ Map<String, String> portalsBefore = new LinkedHashMap<>(portals);
+ if (targetType == 'S') {
+ statements.remove(name);
+ portals.entrySet().removeIf(entry -> entry.getValue().equals(name));
+ } else {
+ portals.remove(name);
+ }
+ return record(statementsBefore, portalsBefore);
+ }
+
+ synchronized void confirm(Mutation mutation) {
+ journal.removeIf(entry -> entry.mutation().equals(mutation));
+ }
+
+ synchronized void rejectFrom(Mutation mutation) {
+ int rejectedIndex = -1;
+ for (int i = 0; i < journal.size(); i++) {
+ if (journal.get(i).mutation().equals(mutation)) {
+ rejectedIndex = i;
+ break;
+ }
+ }
+ if (rejectedIndex < 0) {
+ return;
+ }
+
+ // The caller rejects one pipelined mutation at a time, in journal order (see
+ // BackendResponseCoordinator#discardQueuedOperationsBeforeSync), so this only ever needs to
+ // undo the rejected mutation's own effect. A later mutation that built on it (e.g. a Bind
+ // naming the statement this Parse defined) gets its own rejectFrom call once the coordinator
+ // reaches it; blindly restoring the whole map back to this entry's "before" would also wipe
+ // out any later, independent mutation that happened to run first (see
+ // rejectingAnEarlierMutationRetainsLaterSyncCycleMutation). The entry's own "after" snapshot
+ // is used rather than the live map or the next entry's "before": both go stale once an
+ // earlier reject's cascading portal cleanup has already touched a key this mutation never
+ // itself changed.
+ JournalEntry rejected = journal.get(rejectedIndex);
+ revertUnchangedSince(statements, rejected.statementsBefore(), rejected.statementsAfter());
+ revertUnchangedSince(portals, rejected.portalsBefore(), rejected.portalsAfter());
+ // A statement this mutation defined may just have been reverted away; drop any portal now
+ // pointing at a statement that no longer exists, the same cleanup stageClose('S', ...) does.
+ portals.entrySet().removeIf(entry -> !statements.containsKey(entry.getValue()));
+
+ journal.remove(rejectedIndex);
+ }
+
+ /**
+ * Reverts {@code live}'s entries for keys the rejected mutation changed (where {@code before}
+ * and {@code after} differ), but only where the live map still holds the value the mutation set
+ * ({@code after}): a key a later mutation has since changed again is left alone, since that
+ * later mutation now owns it and will be reverted by its own {@link #rejectFrom} call if needed.
+ */
+ private static <K, V> void revertUnchangedSince(Map<K, V> live, Map<K, V> before, Map<K, V> after) {
+ Set<K> touchedKeys = new LinkedHashSet<>(before.keySet());
+ touchedKeys.addAll(after.keySet());
+ for (K key : touchedKeys) {
+ V beforeValue = before.get(key);
+ V afterValue = after.get(key);
+ if (Objects.equals(beforeValue, afterValue)) {
+ continue;
+ }
+ if (!Objects.equals(live.get(key), afterValue)) {
+ continue;
+ }
+ if (beforeValue == null) {
+ live.remove(key);
+ } else {
+ live.put(key, beforeValue);
+ }
+ }
+ }
+
+ synchronized Optional<CopyStatementParser.S3Statement> statement(String statementName) {
+ return Optional.ofNullable(statements.get(statementName));
+ }
+
+ synchronized Optional<CopyStatementParser.S3Statement> portal(String portalName) {
+ String statementName = portals.get(portalName);
+ return Optional.ofNullable(statementName).map(statements::get);
+ }
+
+ synchronized void clearPortals() {
+ portals.clear();
+ }
+
+ synchronized void transactionEnded() {
+ if (journal.isEmpty()) {
+ portals.clear();
+ return;
+ }
+
+ Map<String, String> expiredPortals = journal.get(0).portalsBefore();
+ removeUnchangedExpiredPortals(portals, expiredPortals);
+ for (int i = 0; i < journal.size(); i++) {
+ JournalEntry entry = journal.get(i);
+ Map<String, String> portalsBefore = new LinkedHashMap<>(entry.portalsBefore());
+ removeUnchangedExpiredPortals(portalsBefore, expiredPortals);
+ journal.set(i, new JournalEntry(entry.mutation(), entry.statementsBefore(), portalsBefore,
+ entry.statementsAfter(), entry.portalsAfter()));
+ }
+ }
+
+ synchronized void clear() {
+ statements.clear();
+ portals.clear();
+ journal.clear();
+ }
+
+ /**
+ * Journals a mutation with both its "before" snapshots (passed in, captured prior to the
+ * caller's own action) and its "after" snapshots, captured here immediately once that action has
+ * been applied. Storing "after" explicitly, rather than reconstructing it later from the next
+ * entry's "before" or the live maps, keeps it a fixed historical fact: those two proxies go
+ * stale once an earlier {@link #rejectFrom} has already run its own cascading portal cleanup.
+ */
+ private Mutation record(Map<String, CopyStatementParser.S3Statement> statementsBefore,
+ Map<String, String> portalsBefore) {
+ Mutation mutation = new Mutation(nextMutationId++);
+ journal.add(new JournalEntry(
+ mutation,
+ statementsBefore,
+ portalsBefore,
+ new LinkedHashMap<>(statements),
+ new LinkedHashMap<>(portals)));
+ return mutation;
+ }
+
+ private static void removeUnchangedExpiredPortals(Map<String, String> candidates,
+ Map<String, String> expiredPortals) {
+ candidates.entrySet().removeIf(entry -> expiredPortals.containsKey(entry.getKey())
+ && Objects.equals(expiredPortals.get(entry.getKey()), entry.getValue()));
+ }
+
+ record Mutation(long id) {
+ }
+
+ private record JournalEntry(
+ Mutation mutation,
+ Map<String, CopyStatementParser.S3Statement> statementsBefore,
+ Map<String, String> portalsBefore,
+ Map<String, CopyStatementParser.S3Statement> statementsAfter,
+ Map<String, String> portalsAfter) {
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java
new file mode 100644
index 0000000000..09d67cf398
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3Exchange.java
@@ -0,0 +1,275 @@
+package io.github.hectorvent.floci.services.redshift.proxy;
+
+import io.github.hectorvent.floci.services.s3.S3Service;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.Socket;
+
+final class ExtendedS3Exchange {
+
+ private record CopyResult(boolean succeeded, byte[] deferredReadyForQuery) {
+ }
+
+ private ExtendedS3Exchange() {
+ }
+
+ static void execute(Socket client, Socket backend,
+ PostgresWireDecoder.FrontendMessage executeFrame,
+ CopyStatementParser.S3Statement statement, S3Service s3Service,
+ BackendResponseCoordinator coordinator, BackendResponseCoordinator.Ticket ticket) throws IOException {
+ BackendResponseCoordinator.GateResult gate;
+ try {
+ gate = coordinator.awaitExtendedExecuteTurn(ticket);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while awaiting the Extended Query Execute turn", e);
+ }
+ if (gate != BackendResponseCoordinator.GateResult.READY) {
+ return;
+ }
+
+ boolean failed = true;
+ byte[] deferredReadyForQuery = null;
+ try {
+ switch (statement) {
+ case CopyStatementParser.S3CopyFrom copy -> {
+ CopyResult result = runCopy(client, backend, executeFrame, copy, s3Service, coordinator);
+ failed = !result.succeeded();
+ deferredReadyForQuery = result.deferredReadyForQuery();
+ }
+ case CopyStatementParser.S3Unload unload -> {
+ CopyResult result = runUnload(client, backend, executeFrame, unload, s3Service, coordinator);
+ failed = !result.succeeded();
+ deferredReadyForQuery = result.deferredReadyForQuery();
+ }
+ }
+ } catch (IOException | RuntimeException e) {
+ closeQuietly(client);
+ closeQuietly(backend);
+ throw e;
+ } finally {
+ coordinator.completeOwnedExecute(ticket, failed);
+ }
+ if (deferredReadyForQuery != null) {
+ coordinator.onBackendFrame('Z', deferredReadyForQuery);
+ }
+ }
+
+ private static CopyResult runCopy(Socket client, Socket backend,
+ PostgresWireDecoder.FrontendMessage executeFrame,
+ CopyStatementParser.S3CopyFrom spec, S3Service s3Service,
+ BackendResponseCoordinator coordinator) throws IOException {
+ OutputStream backendOut = backend.getOutputStream();
+ backendOut.write(executeFrame.toPacketBytes());
+ backendOut.flush();
+ PostgresWireDecoder.FrontendMessage sync = readClientSync(client, backendOut);
+
+ PostgresWireDecoder decoder = new PostgresWireDecoder(backend.getInputStream());
+ PostgresWireDecoder.FrontendMessage first = nextOwnedFrame(client, decoder, coordinator);
+ if (first.type() == 'E') {
+ forward(client, first);
+ forwardClientSyncToBackend(sync, backendOut, coordinator);
+ return new CopyResult(false, null);
+ }
+ if (first.type() != 'G') {
+ throw unexpected(first, "CopyInResponse");
+ }
+
+ S3CopySimulator.CopyInput input;
+ try {
+ input = S3CopySimulator.prepareCopy(spec, s3Service);
+ } catch (S3CopySimulator.S3TransferException e) {
+ S3CopySimulator.writeCopyFail(backendOut, e.getMessage());
+ forwardClientSyncToBackend(sync, backendOut, coordinator);
+ drainExecute(client, decoder, coordinator, false);
+ sendError(client, e.sqlState(), e.getMessage());
+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));
+ }
+
+ try {
+ S3CopySimulator.streamCopyInput(input, backendOut);
+ backendOut.write(new byte[]{'c', 0, 0, 0, 4});
+ backendOut.flush();
+ } catch (RuntimeException | IOException e) {
+ S3CopySimulator.writeCopyFail(backendOut, e.getMessage());
+ forwardClientSyncToBackend(sync, backendOut, coordinator);
+ drainExecute(client, decoder, coordinator, false);
+ String detail = e.getMessage() != null ? e.getMessage() : e.toString();
+ sendError(client, "XX000", "S3 COPY failed: " + detail);
+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));
+ }
+
+ forwardClientSyncToBackend(sync, backendOut, coordinator);
+ PostgresWireDecoder.FrontendMessage terminal = drainExecute(client, decoder, coordinator, true);
+ return new CopyResult(terminal.type() != 'E', null);
+ }
+
+ private static CopyResult runUnload(Socket client, Socket backend,
+ PostgresWireDecoder.FrontendMessage executeFrame,
+ CopyStatementParser.S3Unload spec, S3Service s3Service,
+ BackendResponseCoordinator coordinator) throws IOException {
+ OutputStream backendOut = backend.getOutputStream();
+ backendOut.write(executeFrame.toPacketBytes());
+ backendOut.flush();
+ forwardClientSyncToBackend(client, backendOut, coordinator);
+
+ PostgresWireDecoder decoder = new PostgresWireDecoder(backend.getInputStream());
+ PostgresWireDecoder.FrontendMessage first = nextOwnedFrame(client, decoder, coordinator);
+ if (first.type() == 'E') {
+ forward(client, first);
+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));
+ }
+ if (first.type() != 'H') {
+ throw unexpected(first, "CopyOutResponse");
+ }
+
+ S3CopySimulator.UnloadCollector collector;
+ try {
+ collector = S3CopySimulator.prepareUnload(spec, s3Service);
+ } catch (S3CopySimulator.S3TransferException e) {
+ S3CopySimulator.writeCopyFail(backendOut, e.getMessage());
+ drainExecute(client, decoder, coordinator, false);
+ sendError(client, e.sqlState(), e.getMessage());
+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));
+ }
+
+ try (collector) {
+ while (true) {
+ PostgresWireDecoder.FrontendMessage message = nextOwnedFrame(client, decoder, coordinator);
+ if (message.type() == 'd') {
+ try {
+ collector.accept(message.body());
+ } catch (S3CopySimulator.S3TransferException e) {
+ collector.abort();
+ S3CopySimulator.writeCopyFail(backendOut, e.getMessage());
+ drainExecute(client, decoder, coordinator, false);
+ sendError(client, e.sqlState(), e.getMessage());
+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));
+ }
+ } else if (message.type() == 'c') {
+ continue;
+ } else if (message.type() == 'E') {
+ collector.abort();
+ forward(client, message);
+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));
+ } else if (message.type() == 'C') {
+ try {
+ collector.complete();
+ } catch (S3CopySimulator.S3TransferException e) {
+ S3CopySimulator.writeCopyFail(backendOut, e.getMessage());
+ sendError(client, e.sqlState(), e.getMessage());
+ return new CopyResult(false, drainReadyForQuery(client, decoder, coordinator));
+ }
+ forward(client, message);
+ return new CopyResult(true, null);
+ } else {
+ throw unexpected(message, "CopyData, CopyDone, or CommandComplete");
+ }
+ }
+ }
+ }
+
+ private static PostgresWireDecoder.FrontendMessage drainExecute(Socket client,
+ PostgresWireDecoder decoder, BackendResponseCoordinator coordinator,
+ boolean forwardTerminal) throws IOException {
+ while (true) {
+ PostgresWireDecoder.FrontendMessage message = nextOwnedFrame(client, decoder, coordinator);
+ char type = message.type();
+ if (type == 'C' || type == 'E' || type == 'I' || type == 's') {
+ if (forwardTerminal) {
+ forward(client, message);
+ }
+ return message;
+ }
+ if (type != 'd' && type != 'c') {
+ throw unexpected(message, "CopyData, CopyDone, or an Execute terminal response");
+ }
+ }
+ }
+
+ private static byte[] drainReadyForQuery(Socket client, PostgresWireDecoder decoder,
+ BackendResponseCoordinator coordinator) throws IOException {
+ PostgresWireDecoder.FrontendMessage message = nextOwnedFrame(client, decoder, coordinator);
+ if (message.type() != 'Z') {
+ throw unexpected(message, "ReadyForQuery");
+ }
+ forward(client, message);
+ return message.body();
+ }
+
+ private static PostgresWireDecoder.FrontendMessage nextOwnedFrame(Socket client,
+ PostgresWireDecoder decoder, BackendResponseCoordinator coordinator) throws IOException {
+ while (true) {
+ PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();
+ if (message == null) {
+ throw new IOException("Backend closed during an Extended Query S3 exchange");
+ }
+ char type = message.type();
+ if (type == 'N' || type == 'A' || type == 'S') {
+ coordinator.onBackendFrame(type, message.body());
+ forward(client, message);
+ continue;
+ }
+ if (type == '1' || type == '2' || type == 'T' || type == 't' || type == 'n') {
+ coordinator.onBackendFrame(type, message.body());
+ forward(client, message);
+ continue;
+ }
+ return message;
+ }
+ }
+
+ private static PostgresWireDecoder.FrontendMessage readClientSync(Socket client, OutputStream backendOut)
+ throws IOException {
+ PostgresWireDecoder decoder = new PostgresWireDecoder(client.getInputStream());
+ while (true) {
+ PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();
+ if (message == null) {
+ throw new IOException("Client closed during an Extended Query S3 exchange");
+ }
+ if (message.type() == 'S') {
+ return message;
+ }
+ if (message.type() != 'H') {
+ throw new IOException("Expected Flush or Sync after an Extended Query S3 Execute");
+ }
+ backendOut.write(message.toPacketBytes());
+ backendOut.flush();
+ }
+ }
+
+ private static void forwardClientSyncToBackend(Socket client, OutputStream backendOut,
+ BackendResponseCoordinator coordinator) throws IOException {
+ forwardClientSyncToBackend(readClientSync(client, backendOut), backendOut, coordinator);
+ }
+
+ private static void forwardClientSyncToBackend(PostgresWireDecoder.FrontendMessage sync, OutputStream backendOut,
+ BackendResponseCoordinator coordinator) throws IOException {
+ coordinator.register(BackendResponseCoordinator.Operation.SYNC, null);
+ backendOut.write(sync.toPacketBytes());
+ backendOut.flush();
+ }
+
+ private static IOException unexpected(PostgresWireDecoder.FrontendMessage message, String expected) {
+ return new IOException("Expected " + expected + " but backend sent " + message.type());
+ }
+
+ private static void sendError(Socket client, String sqlState, String message) throws IOException {
+ forward(client, new PostgresWireDecoder.FrontendMessage(
+ 'E', S3CopySimulator.errorBody(sqlState, message)));
+ }
+
+ private static void forward(Socket client, PostgresWireDecoder.FrontendMessage message) throws IOException {
+ OutputStream out = client.getOutputStream();
+ out.write(message.toPacketBytes());
+ out.flush();
+ }
+
+ private static void closeQuietly(Socket socket) {
+ try {
+ socket.close();
+ } catch (IOException ignored) {
+ // The original protocol failure is more useful than a secondary socket-close failure.
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoder.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoder.java
index b6bc43f358..b650f4a2ba 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoder.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoder.java
@@ -1,11 +1,15 @@
package io.github.hectorvent.floci.services.redshift.proxy;
+import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Objects;
+import java.util.function.IntPredicate;
/**
* Frames the PostgreSQL wire-protocol byte stream sent by the frontend (client).
@@ -50,7 +54,7 @@ public class PostgresWireDecoder {
* @throws IOException on an I/O error, a length below 4, or a length above {@link #MAX_MESSAGE_BYTES}.
*/
public FrontendMessage nextMessage() throws IOException {
- return nextMessage(null);
+ return nextMessage(null, ignored -> true);
}
/**
@@ -66,6 +70,21 @@ public class PostgresWireDecoder {
* @throws IOException on an I/O error, a length below 4, or a length above {@link #MAX_MESSAGE_BYTES}.
*/
public FrontendMessage nextMessage(OutputStream passthroughOut) throws IOException {
+ return nextMessage(passthroughOut, type -> type == 'Q');
+ }
+
+ /**
+ * Read the next frontend message, selecting which message types should be retained for
+ * inspection. Unselected messages are streamed to {@code passthroughOut} when it is non-null.
+ *
+ * @param passthroughOut destination for opaque or oversized frames, or {@code null} to buffer
+ * @param inspectType predicate identifying message types whose bodies should be retained
+ * @return the message, or {@code null} on a clean end-of-stream between messages.
+ * @throws EOFException if EOF is hit inside the length field or body.
+ * @throws IOException on an I/O error, a length below 4, or a length above {@link #MAX_MESSAGE_BYTES}
+ */
+ public FrontendMessage nextMessage(OutputStream passthroughOut, IntPredicate inspectType) throws IOException {
+ Objects.requireNonNull(inspectType, "inspectType must not be null");
betweenMessages = true;
int typeByte = in.read();
@@ -85,7 +104,7 @@ public class PostgresWireDecoder {
throw new IOException("Invalid message length: " + length);
}
- if (passthroughOut != null && (type != 'Q' || length > MAX_MESSAGE_BYTES)) {
+ if (passthroughOut != null && (!inspectType.test(type) || length > MAX_MESSAGE_BYTES)) {
passthroughOut.write(typeByte);
passthroughOut.write(lengthBytes);
int remaining = length - 4;
@@ -117,6 +136,122 @@ public class PostgresWireDecoder {
return new FrontendMessage(type, body);
}
+ /** Decode a Parse ({@code 'P'}) message body. */
+ public ParseMessage decodeParse(FrontendMessage message) throws IOException {
+ BodyCursor cursor = cursorFor(message, 'P');
+ String statementName = cursor.readCString();
+ String sql = cursor.readCString();
+ int parameterCount = cursor.readInt16();
+ List<Integer> parameterTypeOids = new ArrayList<>(parameterCount);
+ for (int i = 0; i < parameterCount; i++) {
+ parameterTypeOids.add(cursor.readInt32());
+ }
+ cursor.requireEnd();
+ return new ParseMessage(statementName, sql, List.copyOf(parameterTypeOids));
+ }
+
+ /** Decode the leading names of a Bind ({@code 'B'}) message body. */
+ public BindMessage decodeBind(FrontendMessage message) throws IOException {
+ BodyCursor cursor = cursorFor(message, 'B');
+ String portalName = cursor.readCString();
+ String statementName = cursor.readCString();
+ return new BindMessage(portalName, statementName);
+ }
+
+ /** Decode an Execute ({@code 'E'}) message body. */
+ public ExecuteMessage decodeExecute(FrontendMessage message) throws IOException {
+ BodyCursor cursor = cursorFor(message, 'E');
+ String portalName = cursor.readCString();
+ int maxRows = cursor.readInt32();
+ cursor.requireEnd();
+ return new ExecuteMessage(portalName, maxRows);
+ }
+
+ /** Decode a Describe ({@code 'D'}) message body. */
+ public TargetMessage decodeDescribe(FrontendMessage message) throws IOException {
+ return decodeTarget(message, 'D');
+ }
+
+ /** Decode a Close ({@code 'C'}) message body. */
+ public TargetMessage decodeClose(FrontendMessage message) throws IOException {
+ return decodeTarget(message, 'C');
+ }
+
+ /** Encode a rewritten Parse ({@code 'P'}) message, retaining its name and parameter OIDs. */
+ public static byte[] encodeParse(ParseMessage parse, String sql) {
+ Objects.requireNonNull(parse, "parse must not be null");
+ String statementName = Objects.requireNonNull(parse.statementName(), "statementName must not be null");
+ String rewrittenSql = Objects.requireNonNull(sql, "sql must not be null");
+ List<Integer> parameterTypeOids = Objects.requireNonNull(
+ parse.parameterTypeOids(), "parameterTypeOids must not be null");
+ if (parameterTypeOids.size() > 0xFFFF) {
+ throw new IllegalArgumentException("Too many PostgreSQL Parse parameter type OIDs: "
+ + parameterTypeOids.size());
+ }
+
+ ByteArrayOutputStream body = new ByteArrayOutputStream();
+ writeCString(body, statementName);
+ writeCString(body, rewrittenSql);
+ writeInt16(body, parameterTypeOids.size());
+ for (Integer oid : parameterTypeOids) {
+ writeInt32(body, Objects.requireNonNull(oid, "parameter type OID must not be null"));
+ }
+
+ byte[] bodyBytes = body.toByteArray();
+ int length = bodyBytes.length + 4;
+ byte[] packet = new byte[bodyBytes.length + 5];
+ packet[0] = 'P';
+ packet[1] = (byte) ((length >> 24) & 0xFF);
+ packet[2] = (byte) ((length >> 16) & 0xFF);
+ packet[3] = (byte) ((length >> 8) & 0xFF);
+ packet[4] = (byte) (length & 0xFF);
+ System.arraycopy(bodyBytes, 0, packet, 5, bodyBytes.length);
+ return packet;
+ }
+
+ private TargetMessage decodeTarget(FrontendMessage message, char expectedType) throws IOException {
+ BodyCursor cursor = cursorFor(message, expectedType);
+ int targetByte = cursor.readByte();
+ if (targetByte != 'S' && targetByte != 'P') {
+ throw new IOException("Invalid PostgreSQL " + expectedType + " target type: " + targetByte);
+ }
+ String name = cursor.readCString();
+ cursor.requireEnd();
+ return new TargetMessage((char) targetByte, name);
+ }
+
+ private static BodyCursor cursorFor(FrontendMessage message, char expectedType) throws IOException {
+ if (message == null) {
+ throw new IOException("PostgreSQL message is missing");
+ }
+ if (message.type() != expectedType) {
+ throw new IOException("Expected PostgreSQL message type " + expectedType
+ + " but received " + message.type());
+ }
+ if (message.body() == null) {
+ throw new IOException("PostgreSQL message body was streamed and is unavailable");
+ }
+ return new BodyCursor(message.body());
+ }
+
+ private static void writeCString(ByteArrayOutputStream out, String value) {
+ byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+ out.writeBytes(bytes);
+ out.write(0);
+ }
+
+ private static void writeInt16(ByteArrayOutputStream out, int value) {
+ out.write((value >> 8) & 0xFF);
+ out.write(value & 0xFF);
+ }
+
+ private static void writeInt32(ByteArrayOutputStream out, int value) {
+ out.write((value >> 24) & 0xFF);
+ out.write((value >> 16) & 0xFF);
+ out.write((value >> 8) & 0xFF);
+ out.write(value & 0xFF);
+ }
+
/** Read exactly {@code n} bytes or throw {@link EOFException}. */
private byte[] readFully(int n) throws IOException {
byte[] buf = new byte[n];
@@ -131,6 +266,62 @@ public class PostgresWireDecoder {
return buf;
}
+ private static final class BodyCursor {
+ private final byte[] body;
+ private int offset;
+
+ private BodyCursor(byte[] body) {
+ this.body = Objects.requireNonNull(body, "body must not be null");
+ }
+
+ private String readCString() throws IOException {
+ int end = offset;
+ while (end < body.length && body[end] != 0) {
+ end++;
+ }
+ if (end == body.length) {
+ throw new IOException("PostgreSQL message contains an unterminated string");
+ }
+ String value = new String(body, offset, end - offset, StandardCharsets.UTF_8);
+ offset = end + 1;
+ return value;
+ }
+
+ private int readByte() throws IOException {
+ requireRemaining(1);
+ return body[offset++] & 0xFF;
+ }
+
+ private int readInt16() throws IOException {
+ requireRemaining(2);
+ int value = ((body[offset] & 0xFF) << 8) | (body[offset + 1] & 0xFF);
+ offset += 2;
+ return value;
+ }
+
+ private int readInt32() throws IOException {
+ requireRemaining(4);
+ int value = ((body[offset] & 0xFF) << 24)
+ | ((body[offset + 1] & 0xFF) << 16)
+ | ((body[offset + 2] & 0xFF) << 8)
+ | (body[offset + 3] & 0xFF);
+ offset += 4;
+ return value;
+ }
+
+ private void requireEnd() throws IOException {
+ if (offset != body.length) {
+ throw new IOException("PostgreSQL message has " + (body.length - offset) + " trailing bytes");
+ }
+ }
+
+ private void requireRemaining(int count) throws IOException {
+ if (body.length - offset < count) {
+ throw new EOFException("PostgreSQL message body ended mid-field");
+ }
+ }
+ }
+
/** Encode an SQL string as a Simple Query ({@code 'Q'}) packet with a NUL-terminated body. */
public static byte[] encodeQuery(String sql) {
if (sql == null) {
@@ -189,4 +380,16 @@ public class PostgresWireDecoder {
return packet;
}
}
+
+ public record ParseMessage(String statementName, String sql, List<Integer> parameterTypeOids) {
+ }
+
+ public record BindMessage(String portalName, String statementName) {
+ }
+
+ public record ExecuteMessage(String portalName, int maxRows) {
+ }
+
+ public record TargetMessage(char targetType, String name) {
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
index db9c9bcf5c..5b07d11637 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
@@ -9,25 +9,23 @@ import java.io.OutputStream;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.LockSupport;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.IntConsumer;
/**
- * Replaces the transparent {@code bridge()} for Redshift connections so Simple Query ({@code 'Q'})
- * traffic can be inspected before it reaches the backing PostgreSQL container.
+ * Replaces the transparent {@code bridge()} for Redshift connections so SQL in Simple and Extended
+ * Query traffic can be inspected before it reaches the backing PostgreSQL container.
*
* <p><b>Model.</b> Backend&rarr;client is a byte pump on a virtual thread. Client&rarr;backend is a
- * framed loop: every frontend message is forwarded opaque except a {@code 'Q'}, whose SQL is run
- * through {@link RedshiftSqlInterceptor#rewrite} and re-encoded only if it changed.
+ * framed loop: SQL-bearing Query and Parse frames can be rewritten, while other frontend messages
+ * are forwarded with their original bytes.
*
* <p><b>Backend ownership.</b> A future interceptor that injects its own query and reads the reply
* (the S3 COPY simulator) must own the backend exclusively and only start when no earlier request
* is still being answered. {@link #backendLock} serialises pump reads against such an exchange, and
- * {@link #outstandingResponses} plus {@link #pumpBetweenMessages} gate it to a safe boundary. If the
- * backend cannot be caught idle within {@link #PUMP_PARK_WAIT_MS}, the exchange is skipped and the
- * original {@code 'Q'} is forwarded (fail-open).
+ * the response coordinator plus {@link #pumpBetweenMessages} gate it to a safe boundary. If the
+ * backend cannot be caught idle within {@link #PUMP_PARK_WAIT_MS}, a Simple Query exchange is
+ * skipped and the original {@code 'Q'} is forwarded (fail-open).
*/
public class RedshiftInterceptingBridge {
@@ -43,13 +41,12 @@ public class RedshiftInterceptingBridge {
private final Socket client;
private final Socket backend;
private final S3Service s3Service;
+ private final ExtendedQuerySession session = new ExtendedQuerySession();
+ private final BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
private final ReentrantLock backendLock = new ReentrantLock(true);
private volatile boolean pumpBetweenMessages = true;
- private final AtomicInteger outstandingResponses = new AtomicInteger(0);
private volatile boolean pumpFinished = false;
- /** Transaction-status byte from the backend's most recent {@code ReadyForQuery}; {@code 'I'} until one is seen. */
- private volatile char lastBackendStatus = 'I';
public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service) {
this.client = client;
@@ -76,7 +73,8 @@ public class RedshiftInterceptingBridge {
while (true) {
PostgresWireDecoder.FrontendMessage msg;
try {
- msg = decoder.nextMessage();
+ msg = decoder.nextMessage(backendOut, type -> type == 'Q' || type == 'P'
+ || type == 'B' || type == 'D' || type == 'E' || type == 'C' || type == 'S');
} catch (SocketTimeoutException e) {
if (decoder.isBetweenMessages()) {
continue;
@@ -88,66 +86,212 @@ public class RedshiftInterceptingBridge {
break;
}
- if (!msg.isQuery()) {
- if (msg.type() == 'S') {
- outstandingResponses.incrementAndGet();
- }
- backendOut.write(msg.toPacketBytes());
- backendOut.flush();
- if (msg.type() == 'X') {
+ if (msg.body() == null) {
+ if (msg.type() == 'P') {
+ session.clear();
+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, null);
+ } else if (msg.type() == 'B') {
+ session.clearPortals();
+ coordinator.register(BackendResponseCoordinator.Operation.BIND, null);
+ } else if (msg.type() == 'D') {
+ coordinator.register(BackendResponseCoordinator.Operation.DESCRIBE_PORTAL, null);
+ } else if (msg.type() == 'E') {
+ session.clearPortals();
+ coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);
+ } else if (msg.type() == 'C') {
+ coordinator.register(BackendResponseCoordinator.Operation.CLOSE, null);
+ } else if (msg.type() == 'X') {
break;
}
continue;
}
-
- String sql = msg.getSql();
-
- CopyStatementParser.S3Statement parsed = null;
- try {
- parsed = CopyStatementParser.parse(sql);
- } catch (RuntimeException e) {
- LOG.warnv("CopyStatementParser failed, forwarding original query: {0}", e.getMessage());
- }
-
- if (parsed != null) {
- CopyStatementParser.S3Statement statement = parsed;
- boolean intercepted = runWithBackendOwned(() -> switch (statement) {
- case CopyStatementParser.S3CopyFrom c ->
- S3CopySimulator.runCopyFrom(client, backend, c, s3Service, lastBackendStatus,
- status -> lastBackendStatus = (char) status);
- case CopyStatementParser.S3Unload u ->
- S3CopySimulator.runUnload(client, backend, u, s3Service, lastBackendStatus,
- status -> lastBackendStatus = (char) status);
- });
- if (intercepted) {
- continue;
- }
- }
-
- byte[] toBackend = msg.toPacketBytes();
- try {
- String rewritten = RedshiftSqlInterceptor.rewrite(sql);
- if (rewritten != sql) {
- toBackend = PostgresWireDecoder.encodeQuery(rewritten);
+ switch (msg.type()) {
+ case 'Q' -> handleSimpleQuery(msg, backendOut);
+ case 'P' -> handleParse(decoder, msg, backendOut);
+ case 'B' -> handleBind(decoder, msg, backendOut);
+ case 'D' -> handleDescribe(decoder, msg, backendOut);
+ case 'E' -> handleExecute(decoder, msg, backendOut);
+ case 'C' -> handleClose(decoder, msg, backendOut);
+ case 'S' -> {
+ coordinator.register(BackendResponseCoordinator.Operation.SYNC, null);
+ write(backendOut, msg.toPacketBytes());
}
- } catch (RuntimeException e) {
- LOG.warnv("RedshiftSqlInterceptor failed, forwarding original query: {0}", e.getMessage());
- toBackend = msg.toPacketBytes();
+ default -> write(backendOut, msg.toPacketBytes());
}
- outstandingResponses.incrementAndGet();
- backendOut.write(toBackend);
- backendOut.flush();
}
} catch (IOException e) {
LOG.debugv(e, "RedshiftInterceptingBridge client loop ended");
} catch (Exception e) {
LOG.warnv(e, "Unexpected error in RedshiftInterceptingBridge");
} finally {
+ coordinator.close();
closeQuietly(client, "client");
closeQuietly(backend, "backend");
}
}
+ private void handleSimpleQuery(PostgresWireDecoder.FrontendMessage message, OutputStream backendOut)
+ throws IOException {
+ String sql = message.getSql();
+ CopyStatementParser.S3Statement parsed = parseS3Statement(sql);
+ if (parsed != null) {
+ CopyStatementParser.S3Statement statement = parsed;
+ boolean intercepted = runWithBackendOwned(() -> switch (statement) {
+ case CopyStatementParser.S3CopyFrom copy -> S3CopySimulator.runCopyFrom(
+ client, backend, copy, s3Service, coordinator.lastReadyStatus(),
+ status -> coordinator.onBackendFrame('Z', new byte[]{(byte) status}));
+ case CopyStatementParser.S3Unload unload -> S3CopySimulator.runUnload(
+ client, backend, unload, s3Service, coordinator.lastReadyStatus(),
+ status -> coordinator.onBackendFrame('Z', new byte[]{(byte) status}));
+ });
+ if (intercepted) {
+ return;
+ }
+ }
+
+ byte[] packet = message.toPacketBytes();
+ try {
+ String rewritten = RedshiftSqlInterceptor.rewrite(sql);
+ if (!rewritten.equals(sql)) {
+ packet = PostgresWireDecoder.encodeQuery(rewritten);
+ }
+ } catch (RuntimeException e) {
+ LOG.warnv("RedshiftSqlInterceptor failed, forwarding original query: {0}", e.getMessage());
+ }
+ coordinator.register(BackendResponseCoordinator.Operation.SIMPLE_QUERY, null);
+ write(backendOut, packet);
+ }
+
+ private void handleParse(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,
+ OutputStream backendOut) throws IOException {
+ PostgresWireDecoder.ParseMessage parse = decoder.decodeParse(message);
+ CopyStatementParser.S3Statement statement = parse.parameterTypeOids().isEmpty()
+ ? parseS3Statement(parse.sql()) : null;
+ ExtendedQuerySession.Mutation mutation = session.stageParse(parse.statementName(), statement);
+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, mutation);
+
+ String rewritten = parse.sql();
+ if (statement instanceof CopyStatementParser.S3CopyFrom copy) {
+ rewritten = S3CopySimulator.copyBackendSql(copy);
+ } else if (statement instanceof CopyStatementParser.S3Unload unload) {
+ rewritten = S3CopySimulator.unloadBackendSql(unload);
+ } else {
+ try {
+ rewritten = RedshiftSqlInterceptor.rewrite(parse.sql());
+ } catch (RuntimeException e) {
+ LOG.warnv("RedshiftSqlInterceptor failed, forwarding original Parse: {0}", e.getMessage());
+ }
+ }
+ byte[] packet = rewritten.equals(parse.sql())
+ ? message.toPacketBytes() : PostgresWireDecoder.encodeParse(parse, rewritten);
+ write(backendOut, packet);
+ }
+
+ private void handleBind(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,
+ OutputStream backendOut) throws IOException {
+ PostgresWireDecoder.BindMessage bind = decoder.decodeBind(message);
+ ExtendedQuerySession.Mutation mutation = session.stageBind(bind.portalName(), bind.statementName());
+ coordinator.register(BackendResponseCoordinator.Operation.BIND, mutation);
+ write(backendOut, message.toPacketBytes());
+ }
+
+ private void handleDescribe(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,
+ OutputStream backendOut) throws IOException {
+ PostgresWireDecoder.TargetMessage describe = decoder.decodeDescribe(message);
+ BackendResponseCoordinator.Operation operation = describe.targetType() == 'S'
+ ? BackendResponseCoordinator.Operation.DESCRIBE_STATEMENT
+ : BackendResponseCoordinator.Operation.DESCRIBE_PORTAL;
+ coordinator.register(operation, null);
+ write(backendOut, message.toPacketBytes());
+ }
+
+ private void handleExecute(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,
+ OutputStream backendOut) throws IOException {
+ PostgresWireDecoder.ExecuteMessage execute = decoder.decodeExecute(message);
+ BackendResponseCoordinator.Ticket ticket = coordinator.register(
+ BackendResponseCoordinator.Operation.EXECUTE, null);
+ CopyStatementParser.S3Statement statement = session.portal(execute.portalName()).orElse(null);
+ if (statement == null) {
+ write(backendOut, message.toPacketBytes());
+ return;
+ }
+ runExtendedWithBackendOwned(message, statement, ticket);
+ }
+
+ private void handleClose(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,
+ OutputStream backendOut) throws IOException {
+ PostgresWireDecoder.TargetMessage close = decoder.decodeClose(message);
+ ExtendedQuerySession.Mutation mutation = session.stageClose(close.targetType(), close.name());
+ coordinator.register(BackendResponseCoordinator.Operation.CLOSE, mutation);
+ write(backendOut, message.toPacketBytes());
+ }
+
+ private CopyStatementParser.S3Statement parseS3Statement(String sql) {
+ try {
+ return CopyStatementParser.parse(sql);
+ } catch (RuntimeException e) {
+ LOG.warnv("CopyStatementParser failed, forwarding original statement: {0}", e.getMessage());
+ return null;
+ }
+ }
+
+ private void runExtendedWithBackendOwned(PostgresWireDecoder.FrontendMessage executeFrame,
+ CopyStatementParser.S3Statement statement, BackendResponseCoordinator.Ticket ticket) throws IOException {
+ BackendResponseCoordinator.GateResult gate;
+ try {
+ gate = coordinator.awaitExtendedExecuteTurn(ticket);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while awaiting Extended Query backend ownership", e);
+ }
+ if (gate != BackendResponseCoordinator.GateResult.READY) {
+ return;
+ }
+
+ long deadlineNanos = System.nanoTime() + PUMP_PARK_WAIT_MS * 1_000_000L;
+ boolean locked = false;
+ while (!pumpFinished && System.nanoTime() < deadlineNanos) {
+ try {
+ long remainingMillis = Math.max(1L,
+ TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()));
+ locked = backendLock.tryLock(Math.min(PUMP_PARK_WAIT_MS, remainingMillis), TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while acquiring Extended Query backend ownership", e);
+ }
+ if (!locked) {
+ continue;
+ }
+ if (pumpBetweenMessages) {
+ break;
+ }
+ backendLock.unlock();
+ locked = false;
+ }
+ if (!locked || !pumpBetweenMessages || pumpFinished) {
+ if (locked) {
+ backendLock.unlock();
+ }
+ throw new IOException("Could not take backend ownership for Extended Query Execute");
+ }
+ try {
+ backend.setSoTimeout(EXCHANGE_READ_TIMEOUT_MS);
+ ExtendedS3Exchange.execute(client, backend, executeFrame, statement, s3Service, coordinator, ticket);
+ } finally {
+ try {
+ backend.setSoTimeout(PUMP_READ_TIMEOUT_MS);
+ } catch (IOException e) {
+ LOG.debugv(e, "could not restore backend read timeout (socket closing)");
+ }
+ backendLock.unlock();
+ }
+ }
+
+ private static void write(OutputStream out, byte[] packet) throws IOException {
+ out.write(packet);
+ out.flush();
+ }
+
/**
* Take exclusive ownership of an idle backend at a wire-message boundary, run {@code exchange},
* then release it. Returns {@code false} if the backend could not be caught idle within
@@ -159,6 +303,18 @@ public class RedshiftInterceptingBridge {
if (pumpFinished) {
return false;
}
+ try {
+ if (!coordinator.awaitIdle(PUMP_READ_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
+ if (System.nanoTime() >= deadlineNanos) {
+ LOG.warn("backend did not go idle in time; forwarding the COPY unintercepted");
+ return false;
+ }
+ continue;
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("interrupted while waiting for backend responses", e);
+ }
boolean locked;
try {
locked = backendLock.tryLock(PUMP_READ_TIMEOUT_MS, TimeUnit.MILLISECONDS);
@@ -175,7 +331,7 @@ public class RedshiftInterceptingBridge {
}
boolean backendBusy;
try {
- backendBusy = !pumpBetweenMessages || outstandingResponses.get() > 0;
+ backendBusy = !pumpBetweenMessages;
if (!backendBusy || pumpFinished) {
backend.setSoTimeout(EXCHANGE_READ_TIMEOUT_MS);
try {
@@ -200,10 +356,7 @@ public class RedshiftInterceptingBridge {
}
private void pumpBackendToClient() {
- WireFrameTracker tracker = new WireFrameTracker(status -> {
- lastBackendStatus = (char) status;
- outstandingResponses.updateAndGet(v -> Math.max(0, v - 1));
- });
+ WireFrameTracker tracker = new WireFrameTracker(coordinator::onBackendFrame);
try {
InputStream backendIn = backend.getInputStream();
OutputStream clientOut = client.getOutputStream();
@@ -252,18 +405,20 @@ public class RedshiftInterceptingBridge {
/**
* Tracks PostgreSQL wire-message boundaries in a byte stream without buffering it. Each backend
* message is {@code type(1) . length(int32, includes itself) . body}. {@link #betweenMessages()}
- * is true exactly when the next byte would begin a new message; {@code onReadyForQuery} fires
- * once per completed {@code 'Z'} with its one-byte transaction-status payload.
+ * is true exactly when the next byte would begin a new message. The callback fires after every
+ * complete frame and retains only the one-byte {@code ReadyForQuery} status payload.
*/
static final class WireFrameTracker {
- private final IntConsumer onReadyForQuery;
+ private final FrameCompletion onFrame;
private int headerBytesSeen = 0;
private final byte[] header = new byte[5];
private long bodyRemaining = 0;
+ private long declaredBodyLength = 0;
private char pendingType = 0;
+ private byte readyStatus;
- WireFrameTracker(IntConsumer onReadyForQuery) {
- this.onReadyForQuery = onReadyForQuery;
+ WireFrameTracker(FrameCompletion onFrame) {
+ this.onFrame = onFrame;
}
boolean betweenMessages() {
@@ -274,9 +429,12 @@ public class RedshiftInterceptingBridge {
for (int i = off; i < off + len; i++) {
if (bodyRemaining > 0) {
int current = buf[i] & 0xFF;
+ if (pendingType == 'Z' && declaredBodyLength == 1) {
+ readyStatus = (byte) current;
+ }
bodyRemaining--;
- if (bodyRemaining == 0 && pendingType == 'Z') {
- onReadyForQuery.accept(current);
+ if (bodyRemaining == 0) {
+ publishCompletion();
}
continue;
}
@@ -286,12 +444,24 @@ public class RedshiftInterceptingBridge {
| ((header[3] & 0xFFL) << 8) | (header[4] & 0xFFL);
pendingType = (char) (header[0] & 0xFF);
bodyRemaining = Math.max(0, msgLen - 4);
+ declaredBodyLength = bodyRemaining;
headerBytesSeen = 0;
- if (bodyRemaining == 0 && pendingType == 'Z') {
- onReadyForQuery.accept('I');
+ if (bodyRemaining == 0) {
+ publishCompletion();
}
}
}
}
+
+ private void publishCompletion() {
+ byte[] body = pendingType == 'Z' && declaredBodyLength == 1
+ ? new byte[]{readyStatus} : new byte[0];
+ onFrame.accept(pendingType, body);
+ }
+ }
+
+ @FunctionalInterface
+ interface FrameCompletion {
+ void accept(char type, byte[] body);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
index a2145c3ccd..98624f8c1e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulator.java
@@ -58,38 +58,51 @@ public final class S3CopySimulator {
private S3CopySimulator() {
}
- /**
- * @param txStatus the transaction-status byte from the client's last {@code ReadyForQuery}
- * ({@code 'I'} idle, {@code 'T'} in a block, {@code 'E'} failed block); a
- * synthesized error reports {@code 'I'} outside a block and {@code 'E'} inside.
- * @return {@code true} when the exchange was handled (success or a clean error sent to the
- * client). Part 4 always handles; the value exists so a later interceptor can decline.
- */
- public static boolean runCopyFrom(Socket client, Socket backend,
- CopyStatementParser.S3CopyFrom spec, S3Service s3,
- char txStatus) throws IOException {
- return runCopyFrom(client, backend, spec, s3, txStatus, null);
+ record CopyInput(CopyStatementParser.S3CopyFrom spec, List<String> keys, S3Service s3) {
}
- public static boolean runCopyFrom(Socket client, Socket backend,
- CopyStatementParser.S3CopyFrom spec, S3Service s3,
- char txStatus, IntConsumer onStatusChange) throws IOException {
+ interface UnloadCollector extends AutoCloseable {
+ void accept(byte[] body) throws IOException;
+
+ void complete() throws IOException;
+
+ void abort();
+
+ @Override
+ void close();
+ }
+
+ static final class S3TransferException extends RuntimeException {
+ private final String sqlState;
+
+ S3TransferException(String sqlState, String message, Throwable cause) {
+ super(message, cause);
+ this.sqlState = sqlState;
+ }
+
+ String sqlState() {
+ return sqlState;
+ }
+ }
+
+ static CopyInput prepareCopy(CopyStatementParser.S3CopyFrom spec, S3Service s3) {
try {
s3.authorizeAnonymousListBucket(spec.bucket());
} catch (AwsException e) {
- LOG.debugv(e, "ListBucket denied for COPY from s3://{0}/{1}", spec.bucket(), spec.keyOrPrefix());
- sendAccessDenied(client, backend, spec, txStatus, onStatusChange);
- return true;
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);
}
List<String> keys;
try {
keys = resolveKeys(spec, s3);
} catch (AwsException e) {
- LOG.warnv(e, "failed to list s3://{0}/{1} for COPY", spec.bucket(), spec.keyOrPrefix());
- sendError(client, backend, SQLSTATE_INTERNAL,
- "S3 COPY could not list s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), txStatus, onStatusChange);
- return true;
+ throw new S3TransferException(SQLSTATE_INTERNAL,
+ "S3 COPY could not list s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);
+ }
+ if (keys.isEmpty()) {
+ throw new S3TransferException(SQLSTATE_INTERNAL,
+ "S3 object s3://" + spec.bucket() + "/" + spec.keyOrPrefix() + " not found", null);
}
try {
@@ -97,19 +110,82 @@ public final class S3CopySimulator {
s3.authorizeAnonymousGetObject(spec.bucket(), key);
}
} catch (AwsException e) {
- LOG.debugv(e, "GetObject denied for COPY from s3://{0}/{1}", spec.bucket(), spec.keyOrPrefix());
- sendAccessDenied(client, backend, spec, txStatus, onStatusChange);
- return true;
+ throw new S3TransferException(SQLSTATE_INSUFFICIENT_PRIVILEGE,
+ "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), e);
}
+ return new CopyInput(spec, List.copyOf(keys), s3);
+ }
- if (keys.isEmpty()) {
- sendError(client, backend, SQLSTATE_INTERNAL,
- "S3 object s3://" + spec.bucket() + "/" + spec.keyOrPrefix() + " not found", txStatus, onStatusChange);
+ static String copyBackendSql(CopyStatementParser.S3CopyFrom spec) {
+ return fabricateCopy(spec);
+ }
+
+ static void streamCopyInput(CopyInput input, OutputStream backendOut) throws IOException {
+ streamObjects(input.spec(), input.s3(), input.keys(), backendOut);
+ }
+
+ static UnloadCollector prepareUnload(CopyStatementParser.S3Unload spec, S3Service s3) {
+ String probeKey = unloadDataKey(spec, 0);
+ try {
+ s3.authorizeAnonymousPutObject(spec.bucket(), probeKey);
+ if (spec.manifest()) {
+ s3.authorizeAnonymousPutObject(spec.bucket(), spec.prefix() + "manifest");
+ }
+ if (!spec.allowOverwrite()) {
+ s3.authorizeAnonymousListBucket(spec.bucket());
+ if (targetPrefixHasObjects(spec, s3)) {
+ throw new S3TransferException(SQLSTATE_INTERNAL,
+ "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()
+ + " is not empty; specify ALLOWOVERWRITE to overwrite", null);
+ }
+ }
+ } catch (AwsException e) {
+ throw new S3TransferException(unloadWriteSqlState(e), unloadWriteMessage(e, spec), e);
+ }
+
+ if (!UNLOAD_HEAP_MIB.tryAcquire(UNLOAD_INITIAL_MIB)) {
+ throw new S3TransferException(SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,
+ "UNLOAD memory budget exhausted; retry shortly", null);
+ }
+ try {
+ return new S3UnloadCollector(spec, s3);
+ } catch (IOException e) {
+ UNLOAD_HEAP_MIB.release(UNLOAD_INITIAL_MIB);
+ throw new S3TransferException(SQLSTATE_INTERNAL, "UNLOAD failed: " + e.getMessage(), e);
+ }
+ }
+
+ static String unloadBackendSql(CopyStatementParser.S3Unload spec) {
+ return fabricateUnloadCopy(spec);
+ }
+
+ /**
+ * @param txStatus the transaction-status byte from the client's last {@code ReadyForQuery}
+ * ({@code 'I'} idle, {@code 'T'} in a block, {@code 'E'} failed block); a
+ * synthesized error reports {@code 'I'} outside a block and {@code 'E'} inside.
+ * @return {@code true} when the exchange was handled (success or a clean error sent to the
+ * client). Part 4 always handles; the value exists so a later interceptor can decline.
+ */
+ public static boolean runCopyFrom(Socket client, Socket backend,
+ CopyStatementParser.S3CopyFrom spec, S3Service s3,
+ char txStatus) throws IOException {
+ return runCopyFrom(client, backend, spec, s3, txStatus, null);
+ }
+
+ public static boolean runCopyFrom(Socket client, Socket backend,
+ CopyStatementParser.S3CopyFrom spec, S3Service s3,
+ char txStatus, IntConsumer onStatusChange) throws IOException {
+ CopyInput input;
+ try {
+ input = prepareCopy(spec, s3);
+ } catch (S3TransferException e) {
+ LOG.debugv(e, "COPY preparation failed for s3://{0}/{1}", spec.bucket(), spec.keyOrPrefix());
+ sendError(client, backend, e.sqlState(), e.getMessage(), txStatus, onStatusChange);
return true;
}
OutputStream backendOut = backend.getOutputStream();
- backendOut.write(PostgresWireDecoder.encodeQuery(fabricateCopy(spec)));
+ backendOut.write(PostgresWireDecoder.encodeQuery(copyBackendSql(spec)));
backendOut.flush();
PostgresWireDecoder backendDecoder = new PostgresWireDecoder(backend.getInputStream());
@@ -142,7 +218,7 @@ public final class S3CopySimulator {
// a CopyFail to the backend, whose ErrorResponse/ReadyForQuery is relayed to the client;
// or, if the backend is unreachable, one synthesized ErrorResponse/ReadyForQuery.
try {
- streamObjects(spec, s3, keys, backendOut);
+ streamCopyInput(input, backendOut);
writeCopyDone(backendOut);
drainToReadyForQuery(backendDecoder, client, onStatusChange);
} catch (RuntimeException | IOException e) {
@@ -266,7 +342,7 @@ public final class S3CopySimulator {
out.flush();
}
- private static void writeCopyFail(OutputStream out, String reason) throws IOException {
+ static void writeCopyFail(OutputStream out, String reason) throws IOException {
byte[] message = (reason == null ? "S3 COPY aborted" : reason).getBytes(StandardCharsets.UTF_8);
out.write('f');
out.write(intBytes(4 + message.length + 1));
@@ -318,12 +394,6 @@ public final class S3CopySimulator {
client.getOutputStream().flush();
}
- private static void sendAccessDenied(Socket client, Socket backend, CopyStatementParser.S3CopyFrom spec,
- char txStatus, IntConsumer onStatusChange) throws IOException {
- sendError(client, backend, SQLSTATE_INSUFFICIENT_PRIVILEGE,
- "S3 access denied for s3://" + spec.bucket() + "/" + spec.keyOrPrefix(), txStatus, onStatusChange);
- }
-
private static void sendError(Socket client, Socket backend, String sqlState, String message,
char txStatus, IntConsumer onStatusChange) throws IOException {
if (txStatus == 'T' && backend != null && !backend.isClosed()) {
@@ -399,7 +469,7 @@ public final class S3CopySimulator {
}
}
- private static byte[] errorBody(String sqlState, String message) {
+ static byte[] errorBody(String sqlState, String message) {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
writeField(bytes, 'S', "ERROR");
writeField(bytes, 'C', sqlState);
@@ -429,58 +499,29 @@ public final class S3CopySimulator {
public static boolean runUnload(Socket client, Socket backend,
CopyStatementParser.S3Unload spec, S3Service s3, char txStatus,
IntConsumer onStatusChange) throws IOException {
-
- String probeKey = unloadDataKey(spec, 0);
- String manifestKey = spec.prefix() + "manifest";
+ UnloadCollector collector;
try {
- s3.authorizeAnonymousPutObject(spec.bucket(), probeKey);
- if (spec.manifest()) {
- s3.authorizeAnonymousPutObject(spec.bucket(), manifestKey);
- }
- } catch (AwsException e) {
- LOG.debugv(e, "PutObject denied for UNLOAD to s3://{0}/{1}", spec.bucket(), spec.prefix());
- sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);
+ collector = prepareUnload(spec, s3);
+ } catch (S3TransferException e) {
+ LOG.debugv(e, "UNLOAD preparation failed for s3://{0}/{1}", spec.bucket(), spec.prefix());
+ sendError(client, backend, e.sqlState(), e.getMessage(), txStatus, onStatusChange);
return true;
}
- try {
- if (!spec.allowOverwrite()) {
- s3.authorizeAnonymousListBucket(spec.bucket());
- if (targetPrefixHasObjects(spec, s3)) {
- sendError(client, backend, SQLSTATE_INTERNAL,
- "S3 prefix s3://" + spec.bucket() + "/" + spec.prefix()
- + " is not empty; specify ALLOWOVERWRITE to overwrite", txStatus, onStatusChange);
- return true;
- }
- }
- } catch (AwsException e) {
- LOG.debugv(e, "bucket check failed for UNLOAD to s3://{0}/{1}", spec.bucket(), spec.prefix());
- sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);
- return true;
- }
-
- if (!UNLOAD_HEAP_MIB.tryAcquire(UNLOAD_INITIAL_MIB)) {
- sendError(client, backend, SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,
- "UNLOAD memory budget exhausted; retry shortly", txStatus, onStatusChange);
- return true;
- }
- int[] heldMib = {UNLOAD_INITIAL_MIB};
- try {
- return runUnloadStreaming(client, backend, spec, s3, txStatus, onStatusChange, heldMib);
- } finally {
- UNLOAD_HEAP_MIB.release(heldMib[0]);
+ try (collector) {
+ return runUnloadStreaming(client, backend, spec, collector, txStatus, onStatusChange);
}
}
private static boolean runUnloadStreaming(Socket client, Socket backend,
- CopyStatementParser.S3Unload spec, S3Service s3, char txStatus,
- IntConsumer onStatusChange, int[] heldMib) throws IOException {
+ CopyStatementParser.S3Unload spec, UnloadCollector collector, char txStatus,
+ IntConsumer onStatusChange) throws IOException {
PostgresWireDecoder.FrontendMessage cmdComplete = null;
PostgresWireDecoder.FrontendMessage readyForQuery = null;
OutputStream backendOut = backend.getOutputStream();
- backendOut.write(PostgresWireDecoder.encodeQuery(fabricateUnloadCopy(spec)));
+ backendOut.write(PostgresWireDecoder.encodeQuery(unloadBackendSql(spec)));
backendOut.flush();
PostgresWireDecoder backendDecoder = new PostgresWireDecoder(backend.getInputStream());
@@ -489,6 +530,7 @@ public final class S3CopySimulator {
first = nextNonAsync(backendDecoder, client);
} catch (IOException e) {
LOG.warnv(e, "backend read failed while awaiting CopyOutResponse");
+ collector.abort();
closeQuietly(backend);
sendError(client, null, SQLSTATE_INTERNAL,
"UNLOAD failed: backend closed or timed out", txStatus, onStatusChange);
@@ -496,6 +538,7 @@ public final class S3CopySimulator {
return true;
}
if (first == null) {
+ collector.abort();
closeQuietly(backend);
sendError(client, null, SQLSTATE_INTERNAL,
"UNLOAD failed: backend closed before COPY started", txStatus, onStatusChange);
@@ -503,217 +546,65 @@ public final class S3CopySimulator {
return true;
}
if (first.type() != 'H') {
- // Backend rejected the SELECT itself. Its ErrorResponse + ReadyForQuery are the client's one response.
+ collector.abort();
forward(client, first);
drainToReadyForQuery(backendDecoder, client, onStatusChange);
return true;
}
- long threshold = spec.maxFileSizeBytes() > 0 ? spec.maxFileSizeBytes() : UNLOAD_TARGET_FILE_BYTES;
- String contentType = spec.gzip() ? "application/gzip" : "text/plain";
- List<String> writtenKeys = new ArrayList<>();
- List<Integer> writtenLengths = new ArrayList<>();
-
- ByteArrayOutputStream sink = new ByteArrayOutputStream();
- OutputStream acc = spec.gzip() ? new GZIPOutputStream(sink) : sink;
- long rawSlice = 0;
- // Data-row bytes in the current slice, excluding a repeated header. Drives the size split so a
- // header-only slice is never cut off on its own.
- long slicePayload = 0;
- long rawTotal = 0;
- boolean lastByteNewline = true;
- boolean warned = false;
- int sliceIndex = 0;
-
- // With HEADER the backend emits the header row once, at the top of the stream. Capture it so
- // it can be repeated at the start of every later slice, the way real Redshift writes one
- // header per output file.
- ByteArrayOutputStream headerBuf = spec.header() ? new ByteArrayOutputStream() : null;
- byte[] headerBytes = null;
- boolean capturingHeader = spec.header();
-
try {
while (true) {
- PostgresWireDecoder.FrontendMessage m = backendDecoder.nextMessage();
- if (m == null) {
- closeAcc(acc, sink);
+ PostgresWireDecoder.FrontendMessage message = backendDecoder.nextMessage();
+ if (message == null) {
+ collector.abort();
closeQuietly(backend);
- if (spec.manifest()) {
- deleteWritten(s3, spec, writtenKeys);
- }
sendError(client, null, SQLSTATE_INTERNAL,
"UNLOAD failed: backend closed mid-stream", txStatus, onStatusChange);
closeQuietly(client);
return true;
}
- char t = m.type();
- if (t == 'd') {
- byte[] body = m.body();
- int dataStart = 0;
- if (capturingHeader) {
- int nl = -1;
- for (int i = 0; i < body.length; i++) {
- if (body[i] == '\n') {
- nl = i;
- break;
- }
- }
- int end = nl >= 0 ? nl + 1 : body.length;
- headerBuf.write(body, 0, end);
- if (nl >= 0) {
- headerBytes = headerBuf.toByteArray();
- headerBuf = null;
- capturingHeader = false;
- }
- dataStart = end;
- } else if (headerBytes != null && sliceIndex > 0 && rawSlice == 0) {
- // First data of a later slice: repeat the captured header row.
- acc.write(headerBytes);
- rawSlice += headerBytes.length;
- }
- acc.write(body, 0, body.length);
- rawSlice += body.length;
- rawTotal += body.length;
- slicePayload += body.length - dataStart;
- if (body.length > 0) {
- lastByteNewline = body[body.length - 1] == '\n';
- }
- if (!warned && rawTotal > UNLOAD_WARN_BYTES) {
- warned = true;
- LOG.warnv("UNLOAD result passed {0} bytes and is buffered a slice at a time "
- + "(bucket={1}, prefix={2})", UNLOAD_WARN_BYTES, spec.bucket(), spec.prefix());
- }
- int wantMib = (int) ((rawSlice * 3) / (1024 * 1024)) + 1;
- while (heldMib[0] < wantMib) {
- if (!UNLOAD_HEAP_MIB.tryAcquire(1)) {
- return abortUnload(client, backend, backendDecoder, acc, sink,
- SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,
- "UNLOAD memory budget exhausted; retry with a smaller result",
- txStatus, onStatusChange);
- }
- heldMib[0]++;
- }
- if (rawTotal > UNLOAD_MAX_TOTAL_BYTES) {
- closeAcc(acc, sink);
- closeQuietly(backend);
- if (spec.manifest()) {
- deleteWritten(s3, spec, writtenKeys);
- }
- sendError(client, null,
- SQLSTATE_PROGRAM_LIMIT_EXCEEDED,
- "UNLOAD result exceeds the " + UNLOAD_MAX_TOTAL_BYTES + "-byte limit",
- txStatus, onStatusChange);
- closeQuietly(client);
- return true;
- }
- if (slicePayload >= threshold && lastByteNewline) {
- byte[] payload = finishSlice(acc, sink);
- String key = unloadDataKey(spec, sliceIndex);
- try {
- s3.authorizeAnonymousPutObject(spec.bucket(), key);
- s3.putObject(spec.bucket(), key, payload, contentType, Map.of());
- } catch (AwsException e) {
- LOG.debugv(e, "UNLOAD slice write to s3://{0}/{1} failed", spec.bucket(), key);
- if (spec.manifest()) {
- deleteWritten(s3, spec, writtenKeys);
- }
- return abortUnload(client, backend, backendDecoder, sink, sink,
- unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);
- }
- writtenKeys.add(key);
- writtenLengths.add(payload.length);
- sliceIndex++;
- int release = heldMib[0] - UNLOAD_INITIAL_MIB;
- if (release > 0) {
- UNLOAD_HEAP_MIB.release(release);
- heldMib[0] = UNLOAD_INITIAL_MIB;
- }
- rawSlice = 0;
- slicePayload = 0;
- lastByteNewline = true;
- sink = new ByteArrayOutputStream();
- acc = spec.gzip() ? new GZIPOutputStream(sink) : sink;
- }
- } else if (t == 'c') {
- // CopyDone, ignore
- } else if (t == 'C') {
- // CommandComplete, hold until we see 'Z'
- cmdComplete = m;
- } else if (t == 'Z') {
- readyForQuery = m;
+ char type = message.type();
+ if (type == 'd') {
+ collector.accept(message.body());
+ } else if (type == 'C') {
+ cmdComplete = message;
+ } else if (type == 'Z') {
+ readyForQuery = message;
break;
- } else if (t == 'E') {
- closeAcc(acc, sink);
- if (spec.manifest()) {
- deleteWritten(s3, spec, writtenKeys);
- }
- forward(client, m);
+ } else if (type == 'E') {
+ collector.abort();
+ forward(client, message);
drainToReadyForQuery(backendDecoder, client, onStatusChange);
return true;
- } else if (t == 'N' || t == 'A' || t == 'S') {
- forward(client, m);
+ } else if (type == 'N' || type == 'A' || type == 'S') {
+ forward(client, message);
}
}
- } catch (AwsException e) {
+ collector.complete();
+ } catch (S3TransferException e) {
LOG.debugv(e, "UNLOAD S3 operation failed during streaming");
- if (spec.manifest()) {
- deleteWritten(s3, spec, writtenKeys);
+ if (readyForQuery != null) {
+ sendError(client, backend, e.sqlState(), e.getMessage(), txStatus, onStatusChange);
+ return true;
+ }
+ if (SQLSTATE_PROGRAM_LIMIT_EXCEEDED.equals(e.sqlState())) {
+ closeQuietly(backend);
+ sendError(client, null, e.sqlState(), e.getMessage(), txStatus, onStatusChange);
+ closeQuietly(client);
+ return true;
}
- return abortUnload(client, backend, backendDecoder, acc, sink,
- unloadWriteSqlState(e), unloadWriteMessage(e, spec), txStatus, onStatusChange);
+ return abortUnload(client, backend, backendDecoder, collector,
+ e.sqlState(), e.getMessage(), txStatus, onStatusChange);
} catch (RuntimeException | IOException e) {
LOG.warnv(e, "UNLOAD streaming failed");
- if (spec.manifest()) {
- deleteWritten(s3, spec, writtenKeys);
- }
String detail = e.getMessage() != null ? e.getMessage() : e.toString();
- return abortUnload(client, backend, backendDecoder, acc, sink,
- SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, txStatus, onStatusChange);
- }
-
- // Flush the final slice. Write it when it carries rows, or when nothing has been written yet
- // (a zero-row UNLOAD still emits one object, header only when HEADER was asked for). A result
- // that ended exactly on a slice boundary leaves slicePayload == 0 with slices already written,
- // so neither an empty nor a header-only trailer is produced.
- byte[] payload = finishSlice(acc, sink);
- if (slicePayload > 0 || writtenKeys.isEmpty()) {
- String key = unloadDataKey(spec, sliceIndex);
- try {
- s3.authorizeAnonymousPutObject(spec.bucket(), key);
- s3.putObject(spec.bucket(), key, payload, contentType, Map.of());
- } catch (AwsException e) {
- LOG.debugv(e, "UNLOAD final slice write to s3://{0}/{1} failed", spec.bucket(), key);
- if (spec.manifest()) {
- deleteWritten(s3, spec, writtenKeys);
- }
- sendError(client, backend, unloadWriteSqlState(e), unloadWriteMessage(e, spec),
- txStatus, onStatusChange);
- return true;
- } catch (RuntimeException e) {
- LOG.warnv(e, "UNLOAD final slice write failed");
- if (spec.manifest()) {
- deleteWritten(s3, spec, writtenKeys);
- }
- String detail = e.getMessage() != null ? e.getMessage() : e.toString();
- sendError(client, backend, SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, txStatus, onStatusChange);
- return true;
- }
- writtenKeys.add(key);
- writtenLengths.add(payload.length);
- }
-
- if (spec.manifest()) {
- try {
- s3.authorizeAnonymousPutObject(spec.bucket(), spec.prefix() + "manifest");
- s3.putObject(spec.bucket(), spec.prefix() + "manifest",
- manifestJson(spec.bucket(), writtenKeys, writtenLengths).getBytes(StandardCharsets.UTF_8),
- "application/json", Map.of());
- } catch (RuntimeException e) {
- LOG.warnv(e, "UNLOAD manifest write failed; removing partial data objects");
- deleteWritten(s3, spec, writtenKeys);
- sendError(client, backend, SQLSTATE_INTERNAL, "UNLOAD manifest write failed", txStatus, onStatusChange);
+ if (readyForQuery != null) {
+ sendError(client, backend, SQLSTATE_INTERNAL,
+ "UNLOAD failed: " + detail, txStatus, onStatusChange);
return true;
}
+ return abortUnload(client, backend, backendDecoder, collector,
+ SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, txStatus, onStatusChange);
}
if (cmdComplete != null) {
@@ -751,6 +642,196 @@ public final class S3CopySimulator {
return sql.toString();
}
+ private static final class S3UnloadCollector implements UnloadCollector {
+ private final CopyStatementParser.S3Unload spec;
+ private final S3Service s3;
+ private final long threshold;
+ private final String contentType;
+ private final List<String> writtenKeys = new ArrayList<>();
+ private final List<Integer> writtenLengths = new ArrayList<>();
+
+ private ByteArrayOutputStream sink = new ByteArrayOutputStream();
+ private OutputStream acc;
+ private ByteArrayOutputStream headerBuf;
+ private byte[] headerBytes;
+ private long rawSlice;
+ private long slicePayload;
+ private long rawTotal;
+ private boolean lastByteNewline = true;
+ private boolean warned;
+ private boolean capturingHeader;
+ private boolean finished;
+ private boolean aborted;
+ private boolean closed;
+ private int sliceIndex;
+ private int heldMib = UNLOAD_INITIAL_MIB;
+
+ private S3UnloadCollector(CopyStatementParser.S3Unload spec, S3Service s3) throws IOException {
+ this.spec = spec;
+ this.s3 = s3;
+ threshold = spec.maxFileSizeBytes() > 0 ? spec.maxFileSizeBytes() : UNLOAD_TARGET_FILE_BYTES;
+ contentType = spec.gzip() ? "application/gzip" : "text/plain";
+ acc = newAccumulator();
+ headerBuf = spec.header() ? new ByteArrayOutputStream() : null;
+ capturingHeader = spec.header();
+ }
+
+ @Override
+ public void accept(byte[] body) throws IOException {
+ ensureOpen();
+ int dataStart = 0;
+ if (capturingHeader) {
+ int newline = -1;
+ for (int i = 0; i < body.length; i++) {
+ if (body[i] == '\n') {
+ newline = i;
+ break;
+ }
+ }
+ int end = newline >= 0 ? newline + 1 : body.length;
+ headerBuf.write(body, 0, end);
+ if (newline >= 0) {
+ headerBytes = headerBuf.toByteArray();
+ headerBuf = null;
+ capturingHeader = false;
+ }
+ dataStart = end;
+ } else if (headerBytes != null && sliceIndex > 0 && rawSlice == 0) {
+ acc.write(headerBytes);
+ rawSlice += headerBytes.length;
+ }
+
+ acc.write(body, 0, body.length);
+ rawSlice += body.length;
+ rawTotal += body.length;
+ slicePayload += body.length - dataStart;
+ if (body.length > 0) {
+ lastByteNewline = body[body.length - 1] == '\n';
+ }
+ if (!warned && rawTotal > UNLOAD_WARN_BYTES) {
+ warned = true;
+ LOG.warnv("UNLOAD result passed {0} bytes and is buffered a slice at a time "
+ + "(bucket={1}, prefix={2})", UNLOAD_WARN_BYTES, spec.bucket(), spec.prefix());
+ }
+ acquireForCurrentSlice();
+ if (rawTotal > UNLOAD_MAX_TOTAL_BYTES) {
+ fail(SQLSTATE_PROGRAM_LIMIT_EXCEEDED,
+ "UNLOAD result exceeds the " + UNLOAD_MAX_TOTAL_BYTES + "-byte limit", null);
+ }
+ if (slicePayload >= threshold && lastByteNewline) {
+ writeCurrentSlice();
+ resetSlice();
+ }
+ }
+
+ @Override
+ public void complete() throws IOException {
+ ensureOpen();
+ byte[] payload = finishSlice(acc, sink);
+ if (slicePayload > 0 || writtenKeys.isEmpty()) {
+ writePayload(payload, sliceIndex);
+ }
+ if (spec.manifest()) {
+ try {
+ String key = spec.prefix() + "manifest";
+ s3.authorizeAnonymousPutObject(spec.bucket(), key);
+ s3.putObject(spec.bucket(), key,
+ manifestJson(spec.bucket(), writtenKeys, writtenLengths)
+ .getBytes(StandardCharsets.UTF_8),
+ "application/json", Map.of());
+ } catch (RuntimeException e) {
+ fail(SQLSTATE_INTERNAL, "UNLOAD manifest write failed", e);
+ }
+ }
+ finished = true;
+ }
+
+ @Override
+ public void abort() {
+ if (aborted || finished) {
+ return;
+ }
+ closeAcc(acc, sink);
+ if (spec.manifest()) {
+ deleteWritten(s3, spec, writtenKeys);
+ }
+ aborted = true;
+ }
+
+ @Override
+ public void close() {
+ if (closed) {
+ return;
+ }
+ if (!finished && !aborted) {
+ abort();
+ }
+ UNLOAD_HEAP_MIB.release(heldMib);
+ heldMib = 0;
+ closed = true;
+ }
+
+ private void acquireForCurrentSlice() {
+ int wantedMib = (int) ((rawSlice * 3) / (1024 * 1024)) + 1;
+ while (heldMib < wantedMib) {
+ if (!UNLOAD_HEAP_MIB.tryAcquire(1)) {
+ fail(SQLSTATE_CONFIGURATION_LIMIT_EXCEEDED,
+ "UNLOAD memory budget exhausted; retry with a smaller result", null);
+ }
+ heldMib++;
+ }
+ }
+
+ private void writeCurrentSlice() throws IOException {
+ byte[] payload = finishSlice(acc, sink);
+ writePayload(payload, sliceIndex);
+ sliceIndex++;
+ }
+
+ private void writePayload(byte[] payload, int index) {
+ String key = unloadDataKey(spec, index);
+ try {
+ s3.authorizeAnonymousPutObject(spec.bucket(), key);
+ s3.putObject(spec.bucket(), key, payload, contentType, Map.of());
+ } catch (AwsException e) {
+ fail(unloadWriteSqlState(e), unloadWriteMessage(e, spec), e);
+ } catch (RuntimeException e) {
+ String detail = e.getMessage() != null ? e.getMessage() : e.toString();
+ fail(SQLSTATE_INTERNAL, "UNLOAD failed: " + detail, e);
+ }
+ writtenKeys.add(key);
+ writtenLengths.add(payload.length);
+ }
+
+ private void resetSlice() throws IOException {
+ int release = heldMib - UNLOAD_INITIAL_MIB;
+ if (release > 0) {
+ UNLOAD_HEAP_MIB.release(release);
+ heldMib = UNLOAD_INITIAL_MIB;
+ }
+ rawSlice = 0;
+ slicePayload = 0;
+ lastByteNewline = true;
+ sink = new ByteArrayOutputStream();
+ acc = newAccumulator();
+ }
+
+ private OutputStream newAccumulator() throws IOException {
+ return spec.gzip() ? new GZIPOutputStream(sink) : sink;
+ }
+
+ private void ensureOpen() {
+ if (finished || aborted || closed) {
+ throw new IllegalStateException("UNLOAD collector is no longer open");
+ }
+ }
+
+ private void fail(String sqlState, String message, Throwable cause) {
+ abort();
+ throw new S3TransferException(sqlState, message, cause);
+ }
+ }
+
/** {@code 404} from the S3 layer means the bucket is absent, not that access was denied. */
private static String unloadWriteSqlState(AwsException e) {
return e.getHttpStatus() == 404 ? SQLSTATE_INTERNAL : SQLSTATE_INSUFFICIENT_PRIVILEGE;
@@ -787,14 +868,12 @@ public final class S3CopySimulator {
}
private static boolean abortUnload(Socket client, Socket backend, PostgresWireDecoder backendDecoder,
- OutputStream acc, ByteArrayOutputStream sink, String sqlState, String message,
+ UnloadCollector collector, String sqlState, String message,
char txStatus, IntConsumer onStatusChange) throws IOException {
- closeAcc(acc, sink);
+ collector.abort();
try {
drainBackendDiscarding(backendDecoder);
} catch (IOException backendGone) {
- // The backend is unreachable, so it cannot be resynchronized: close it and hand the
- // client its one response synthesized, exactly as the backend-EOF branch does.
LOG.debugv(backendGone, "backend unreachable while draining an aborted UNLOAD; synthesizing client error");
closeQuietly(backend);
sendError(client, null, sqlState, message, txStatus, onStatusChange);
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java
index 9fe447dd54..ba0db3196e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftInterceptorIntegrationTest.java
@@ -14,9 +14,11 @@ import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
@@ -31,6 +33,7 @@ import java.util.zip.GZIPOutputStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
@QuarkusTest
@@ -68,24 +71,100 @@ class RedshiftInterceptorIntegrationTest {
}
}
- private static String jdbcUrl(Cluster c) {
+ private static String jdbcUrl(Cluster cluster) {
// Use 127.0.0.1 explicitly instead of c.getEndpoint().getAddress() to avoid UnknownHostException
// in CI environments where floci.emulator.hostname is set to host.docker.internal.
- // preferQueryMode=simple forces pgjdbc to use the simple query protocol ('Q' messages)
- // rather than extended query protocol ('P'/'B'/'E'/'S' messages).
- return "jdbc:postgresql://127.0.0.1:" + c.getEndpoint().getPort() + "/dev?preferQueryMode=simple";
+ return "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort() + "/dev";
+ }
+
+ private static String namedPreparedJdbcUrl(Cluster cluster) {
+ return jdbcUrl(cluster) + "?prepareThreshold=1";
}
private static Connection waitForConnection(Cluster cluster, String username, String password) throws SQLException {
+ return waitForConnection(jdbcUrl(cluster), username, password);
+ }
+
+ private static Connection waitForConnection(String jdbcUrl, String username, String password) throws SQLException {
try {
return Awaitility.await()
.atMost(Duration.ofSeconds(30))
.pollInterval(Duration.ofMillis(500))
.ignoreExceptions()
- .until(() -> DriverManager.getConnection(jdbcUrl(cluster), username, password), Objects::nonNull);
+ .until(() -> DriverManager.getConnection(jdbcUrl, username, password), Objects::nonNull);
} catch (ConditionTimeoutException e) {
- return DriverManager.getConnection(jdbcUrl(cluster), username, password); // throw original
+ return DriverManager.getConnection(jdbcUrl, username, password); // throw original
+ }
+ }
+
+ @Test
+ void preparedDdlCopyAndUnloadUseDefaultExtendedQuery() throws Exception {
+ clusterId = "it-extended-prepared";
+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+ String bucket = "redshift-extended";
+ s3.createBucket(bucket, "us-east-1");
+ s3.putObject(bucket, "copy/data.txt",
+ "1|alice\n2|bob\n".getBytes(StandardCharsets.UTF_8), "text/plain", Map.of());
+
+ try (Connection connection = waitForConnection(cluster, "admin", "Secret123");
+ PreparedStatement ddl = connection.prepareStatement(
+ "CREATE TABLE ext_sales (id int ENCODE az64, name text) "
+ + "DISTSTYLE KEY DISTKEY (id)");
+ PreparedStatement copy = connection.prepareStatement(
+ "COPY ext_sales FROM 's3://redshift-extended/copy/data.txt'");
+ PreparedStatement unload = connection.prepareStatement(
+ "UNLOAD ('select id, name from ext_sales order by id') "
+ + "TO 's3://redshift-extended/unload/' ALLOWOVERWRITE")) {
+ ddl.execute();
+ copy.execute();
+ unload.execute();
+ try (ResultSet rows = connection.createStatement().executeQuery(
+ "SELECT count(*) FROM ext_sales")) {
+ assertTrue(rows.next());
+ assertEquals(2, rows.getInt(1));
+ }
+ }
+
+ List<S3Object> objects = s3.listObjects(bucket, "unload/", null, 100);
+ assertEquals(1, objects.size());
+ assertEquals("1|alice\n2|bob\n", new String(
+ s3.getObject(bucket, objects.get(0).getKey()).getData(), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void namedPreparedCopyAndUnloadCanBeExecutedTwice() throws Exception {
+ clusterId = "it-extended-named";
+ Cluster cluster = service.createCluster(clusterId, "dc2.large", "admin", "Secret123");
+ String bucket = "redshift-extended-named";
+ s3.createBucket(bucket, "us-east-1");
+ s3.putObject(bucket, "copy/data.txt", "7|seven\n".getBytes(StandardCharsets.UTF_8),
+ "text/plain", Map.of());
+
+ try (Connection connection = waitForConnection(
+ namedPreparedJdbcUrl(cluster), "admin", "Secret123")) {
+ connection.createStatement().execute("CREATE TABLE named_sales (id int, name text)");
+ try (PreparedStatement copy = connection.prepareStatement(
+ "COPY named_sales FROM 's3://redshift-extended-named/copy/data.txt'")) {
+ copy.execute();
+ copy.execute();
+ }
+ try (PreparedStatement unload = connection.prepareStatement(
+ "UNLOAD ('select id, name from named_sales order by id') "
+ + "TO 's3://redshift-extended-named/unload/' ALLOWOVERWRITE")) {
+ unload.execute();
+ unload.execute();
+ }
+ try (ResultSet rows = connection.createStatement().executeQuery(
+ "SELECT count(*) FROM named_sales")) {
+ assertTrue(rows.next());
+ assertEquals(2, rows.getInt(1));
+ }
}
+
+ List<S3Object> objects = s3.listObjects(bucket, "unload/", null, 100);
+ assertEquals(1, objects.size());
+ assertEquals("7|seven\n7|seven\n", new String(
+ s3.getObject(bucket, objects.get(0).getKey()).getData(), StandardCharsets.UTF_8));
}
@Test
@@ -212,24 +291,26 @@ class RedshiftInterceptorIntegrationTest {
s3.createBucket(bucket, "us-east-1");
try (Connection c = waitForConnection(cluster, "admin", "Secret123")) {
- c.createStatement().execute("CREATE TABLE tx_test (id int)");
- c.setAutoCommit(false);
- Statement st = c.createStatement();
- st.execute("INSERT INTO tx_test VALUES (1)");
- assertThrows(
- SQLException.class,
- () -> st.execute("COPY tx_test FROM 's3://redshift-copy-it-tx/does/not/exist'"));
- // The backend must now be in the aborted transaction state ('E').
- // Subsequent statements in this transaction block must fail.
- assertThrows(
- SQLException.class,
- () -> st.execute("INSERT INTO tx_test VALUES (2)"));
- c.rollback();
- c.setAutoCommit(true);
- try (ResultSet rs = c.createStatement().executeQuery("SELECT count(*) FROM tx_test")) {
- assertTrue(rs.next());
- assertEquals(0, rs.getInt(1));
- }
+ assertTimeoutPreemptively(Duration.ofSeconds(15), () -> {
+ c.createStatement().execute("CREATE TABLE tx_test (id int)");
+ c.setAutoCommit(false);
+ Statement st = c.createStatement();
+ st.execute("INSERT INTO tx_test VALUES (1)");
+ assertThrows(
+ SQLException.class,
+ () -> st.execute("COPY tx_test FROM 's3://redshift-copy-it-tx/does/not/exist'"));
+ // The backend must now be in the aborted transaction state ('E').
+ // Subsequent statements in this transaction block must fail.
+ assertThrows(
+ SQLException.class,
+ () -> st.execute("INSERT INTO tx_test VALUES (2)"));
+ c.rollback();
+ c.setAutoCommit(true);
+ try (ResultSet rs = c.createStatement().executeQuery("SELECT count(*) FROM tx_test")) {
+ assertTrue(rs.next());
+ assertEquals(0, rs.getInt(1));
+ }
+ });
}
}
@@ -273,12 +354,12 @@ class RedshiftInterceptorIntegrationTest {
c.createStatement().execute("UNLOAD ('select id from g_src order by id') TO 's3://redshift-unload-it-gzip/g/' GZIP");
}
- var objs = s3.listObjects(bucket, "g/", null, 100);
+ List<S3Object> objs = s3.listObjects(bucket, "g/", null, 100);
assertEquals(1, objs.size());
assertTrue(objs.get(0).getKey().endsWith(".gz"), objs.get(0).getKey());
byte[] data = s3.getObject(bucket, objs.get(0).getKey()).getData();
ByteArrayOutputStream raw = new ByteArrayOutputStream();
- try (var in = new GZIPInputStream(new ByteArrayInputStream(data))) {
+ try (InputStream in = new GZIPInputStream(new ByteArrayInputStream(data))) {
in.transferTo(raw);
}
assertEquals("7\n8\n", raw.toString(StandardCharsets.UTF_8));
@@ -297,7 +378,7 @@ class RedshiftInterceptorIntegrationTest {
c.createStatement().execute("UNLOAD ('select id from m_src') TO 's3://redshift-unload-it-manifest/m/' MANIFEST");
}
- var manifest = s3.getObject(bucket, "m/manifest");
+ S3Object manifest = s3.getObject(bucket, "m/manifest");
assertNotNull(manifest);
String json = new String(manifest.getData(), StandardCharsets.UTF_8);
assertTrue(json.contains("\"entries\""), json);
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java
new file mode 100644
index 0000000000..3ef223e0f2
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java
@@ -0,0 +1,181 @@
+package io.github.hectorvent.floci.services.redshift.proxy;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class BackendResponseCoordinatorTest {
+
+ private static final byte[] EMPTY_BODY = new byte[0];
+
+ @Test
+ void parseBindAndDescribeCompleteBeforeExecuteBecomesReady() throws InterruptedException {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+
+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, session.stageParse("s", copy));
+ coordinator.register(BackendResponseCoordinator.Operation.BIND, session.stageBind("p", "s"));
+ coordinator.register(BackendResponseCoordinator.Operation.DESCRIBE_PORTAL, null);
+ BackendResponseCoordinator.Ticket execute = coordinator.register(
+ BackendResponseCoordinator.Operation.EXECUTE, null);
+
+ coordinator.onBackendFrame('1', EMPTY_BODY);
+ coordinator.onBackendFrame('2', EMPTY_BODY);
+ coordinator.onBackendFrame('n', EMPTY_BODY);
+
+ assertEquals(BackendResponseCoordinator.GateResult.READY, coordinator.awaitTurn(execute));
+ assertSame(copy, session.portal("p").orElseThrow());
+ }
+
+ @Test
+ void statementDescribeWaitsPastParameterDescription() throws InterruptedException {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
+ coordinator.register(BackendResponseCoordinator.Operation.DESCRIBE_STATEMENT, null);
+
+ coordinator.onBackendFrame('t', EMPTY_BODY);
+
+ assertFalse(coordinator.awaitIdle(1, TimeUnit.MILLISECONDS));
+ coordinator.onBackendFrame('T', EMPTY_BODY);
+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));
+ }
+
+ @Test
+ void asynchronousFramesDoNotCompleteTheHeadOperation() throws InterruptedException {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, null);
+
+ coordinator.onBackendFrame('N', EMPTY_BODY);
+ coordinator.onBackendFrame('A', EMPTY_BODY);
+ coordinator.onBackendFrame('S', EMPTY_BODY);
+
+ assertFalse(coordinator.awaitIdle(1, TimeUnit.MILLISECONDS));
+ coordinator.onBackendFrame('1', EMPTY_BODY);
+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));
+ }
+
+ @Test
+ void extendedErrorRejectsPipelinedMutationsAndSkipsUntilSync() throws InterruptedException {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, session.stageParse("s", copy));
+ coordinator.register(BackendResponseCoordinator.Operation.BIND, session.stageBind("p", "s"));
+ BackendResponseCoordinator.Ticket execute = coordinator.register(
+ BackendResponseCoordinator.Operation.EXECUTE, null);
+
+ coordinator.onBackendFrame('E', EMPTY_BODY);
+
+ assertEquals(BackendResponseCoordinator.GateResult.SKIPPED_AFTER_ERROR,
+ coordinator.awaitTurn(execute));
+ assertTrue(session.statement("s").isEmpty());
+ assertTrue(session.portal("p").isEmpty());
+ assertTrue(coordinator.isDiscardingUntilSync());
+
+ BackendResponseCoordinator.Ticket skipped = coordinator.register(
+ BackendResponseCoordinator.Operation.EXECUTE, null);
+ assertEquals(BackendResponseCoordinator.GateResult.SKIPPED_AFTER_ERROR,
+ coordinator.awaitTurn(skipped));
+
+ coordinator.register(BackendResponseCoordinator.Operation.SYNC, null);
+ coordinator.onBackendFrame('Z', new byte[]{'I'});
+
+ assertFalse(coordinator.isDiscardingUntilSync());
+ assertEquals('I', coordinator.lastReadyStatus());
+ }
+
+ @Test
+ void errorOnAnOperationWithoutAMutationRejectsLaterPipelinedMutations() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+ coordinator.register(BackendResponseCoordinator.Operation.DESCRIBE_STATEMENT, null);
+ coordinator.register(BackendResponseCoordinator.Operation.BIND, session.stageBind("p", "s"));
+
+ coordinator.onBackendFrame('E', EMPTY_BODY);
+
+ assertTrue(session.portal("p").isEmpty());
+ }
+
+ @Test
+ void simpleQueryErrorCompletesOnlyWhenReadyForQueryArrives() throws InterruptedException {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
+ coordinator.register(BackendResponseCoordinator.Operation.SIMPLE_QUERY, null);
+
+ coordinator.onBackendFrame('E', EMPTY_BODY);
+
+ assertFalse(coordinator.isDiscardingUntilSync());
+ assertFalse(coordinator.awaitIdle(1, TimeUnit.MILLISECONDS));
+
+ coordinator.onBackendFrame('Z', new byte[]{'I'});
+
+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));
+ assertEquals('I', coordinator.lastReadyStatus());
+ }
+
+ @Test
+ void readyForQueryInsideATransactionRetainsPortals() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+ session.confirm(session.stageBind("p", "s"));
+ coordinator.register(BackendResponseCoordinator.Operation.SYNC, null);
+
+ coordinator.onBackendFrame('Z', new byte[]{'T'});
+
+ assertSame(copy, session.portal("p").orElseThrow());
+ assertEquals('T', coordinator.lastReadyStatus());
+ }
+
+ @Test
+ void idleReadyForQueryKeepsPortalsStagedForTheNextPipelinedCycle() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+ session.confirm(session.stageBind("old", "s"));
+ coordinator.register(BackendResponseCoordinator.Operation.SYNC, null);
+ coordinator.register(BackendResponseCoordinator.Operation.BIND, session.stageBind("next", "s"));
+
+ coordinator.onBackendFrame('Z', new byte[]{'I'});
+
+ assertTrue(session.portal("old").isEmpty());
+ assertSame(copy, session.portal("next").orElseThrow());
+
+ coordinator.onBackendFrame('E', EMPTY_BODY);
+ assertTrue(session.portal("old").isEmpty());
+ assertTrue(session.portal("next").isEmpty());
+ }
+
+ @Test
+ void closeWakesABlockedWaiter() throws Exception {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(session);
+ coordinator.register(BackendResponseCoordinator.Operation.PARSE, null);
+ BackendResponseCoordinator.Ticket execute = coordinator.register(
+ BackendResponseCoordinator.Operation.EXECUTE, null);
+ CompletableFuture<BackendResponseCoordinator.GateResult> result = CompletableFuture.supplyAsync(() -> {
+ try {
+ return coordinator.awaitTurn(execute);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ });
+
+ coordinator.close();
+
+ assertEquals(BackendResponseCoordinator.GateResult.CLOSED, result.get(1, TimeUnit.SECONDS));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySessionTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySessionTest.java
new file mode 100644
index 0000000000..7c5c456077
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedQuerySessionTest.java
@@ -0,0 +1,228 @@
+package io.github.hectorvent.floci.services.redshift.proxy;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ExtendedQuerySessionTest {
+
+ @Test
+ void bindMakesAnInterceptedStatementVisibleThroughItsPortal() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+
+ ExtendedQuerySession.Mutation parse = session.stageParse("copy-s", copy);
+ ExtendedQuerySession.Mutation bind = session.stageBind("copy-p", "copy-s");
+
+ assertSame(copy, session.statement("copy-s").orElseThrow());
+ assertSame(copy, session.portal("copy-p").orElseThrow());
+ session.confirm(parse);
+ session.confirm(bind);
+ }
+
+ @Test
+ void ordinaryUnnamedParseAndBindReplaceInterceptedUnnamedMappings() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("", copy));
+ session.confirm(session.stageBind("", ""));
+
+ session.confirm(session.stageParse("", null));
+ session.confirm(session.stageBind("", ""));
+
+ assertTrue(session.statement("").isEmpty());
+ assertTrue(session.portal("").isEmpty());
+ }
+
+ @Test
+ void rejectingAParseRollsBackItAndEveryLaterPipelinedMutation() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ ExtendedQuerySession.Mutation parse = session.stageParse("s", copy);
+ session.stageBind("p", "s");
+
+ session.rejectFrom(parse);
+
+ assertTrue(session.statement("s").isEmpty());
+ assertTrue(session.portal("p").isEmpty());
+ }
+
+ @Test
+ void namedStatementsCoexist() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement first = CopyStatementParser.parse("COPY first FROM 's3://b/one'");
+ CopyStatementParser.S3Statement second = CopyStatementParser.parse("COPY second FROM 's3://b/two'");
+
+ session.confirm(session.stageParse("first", first));
+ session.confirm(session.stageParse("second", second));
+
+ assertSame(first, session.statement("first").orElseThrow());
+ assertSame(second, session.statement("second").orElseThrow());
+ }
+
+ @Test
+ void unnamedPortalRebindingReplacesItsStatement() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement first = CopyStatementParser.parse("COPY first FROM 's3://b/one'");
+ CopyStatementParser.S3Statement second = CopyStatementParser.parse("COPY second FROM 's3://b/two'");
+ session.confirm(session.stageParse("first", first));
+ session.confirm(session.stageParse("second", second));
+ session.confirm(session.stageBind("", "first"));
+
+ session.confirm(session.stageBind("", "second"));
+
+ assertSame(second, session.portal("").orElseThrow());
+ }
+
+ @Test
+ void twoPortalsCanReferenceOneStatement() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+
+ session.confirm(session.stageBind("first", "s"));
+ session.confirm(session.stageBind("second", "s"));
+
+ assertSame(copy, session.portal("first").orElseThrow());
+ assertSame(copy, session.portal("second").orElseThrow());
+ }
+
+ @Test
+ void closingAStatementAlsoClosesItsPortals() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+ session.confirm(session.stageBind("first", "s"));
+ session.confirm(session.stageBind("second", "s"));
+
+ session.confirm(session.stageClose('S', "s"));
+
+ assertTrue(session.statement("s").isEmpty());
+ assertTrue(session.portal("first").isEmpty());
+ assertTrue(session.portal("second").isEmpty());
+ }
+
+ @Test
+ void closingAPortalLeavesItsStatementAndOtherPortals() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+ session.confirm(session.stageBind("first", "s"));
+ session.confirm(session.stageBind("second", "s"));
+
+ session.confirm(session.stageClose('P', "first"));
+
+ assertSame(copy, session.statement("s").orElseThrow());
+ assertTrue(session.portal("first").isEmpty());
+ assertSame(copy, session.portal("second").orElseThrow());
+ }
+
+ @Test
+ void rejectingAStatementCloseRestoresTheStatementAndItsPortals() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+ session.confirm(session.stageBind("first", "s"));
+ session.confirm(session.stageBind("second", "s"));
+
+ ExtendedQuerySession.Mutation close = session.stageClose('S', "s");
+ assertTrue(session.statement("s").isEmpty());
+ assertTrue(session.portal("first").isEmpty());
+ assertTrue(session.portal("second").isEmpty());
+
+ session.rejectFrom(close);
+
+ assertSame(copy, session.statement("s").orElseThrow());
+ assertSame(copy, session.portal("first").orElseThrow());
+ assertSame(copy, session.portal("second").orElseThrow());
+ }
+
+ @Test
+ void rejectingAPortalCloseRestoresOnlyThatPortal() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+ session.confirm(session.stageBind("p", "s"));
+
+ ExtendedQuerySession.Mutation close = session.stageClose('P', "p");
+ session.rejectFrom(close);
+
+ assertSame(copy, session.statement("s").orElseThrow());
+ assertSame(copy, session.portal("p").orElseThrow());
+ }
+
+ @Test
+ void repeatedLookupDoesNotChangeRollbackState() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ ExtendedQuerySession.Mutation parse = session.stageParse("s", copy);
+
+ assertSame(copy, session.statement("s").orElseThrow());
+ assertSame(copy, session.statement("s").orElseThrow());
+ session.rejectFrom(parse);
+
+ assertTrue(session.statement("s").isEmpty());
+ }
+
+ @Test
+ void rejectingAnEarlierMutationRetainsLaterSyncCycleMutation() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+
+ ExtendedQuerySession.Mutation rejected = session.stageParse("invalid", null);
+ ExtendedQuerySession.Mutation retained = session.stageParse("copy", copy);
+
+ session.rejectFrom(rejected);
+
+ assertSame(copy, session.statement("copy").orElseThrow());
+ session.confirm(retained);
+ }
+
+ @Test
+ void transactionEndClearsPortalsButRetainsStatements() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+ session.confirm(session.stageBind("p", "s"));
+
+ session.transactionEnded();
+
+ assertSame(copy, session.statement("s").orElseThrow());
+ assertTrue(session.portal("p").isEmpty());
+ }
+
+ @Test
+ void clearPortalsRetainsStatements() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+ session.confirm(session.stageBind("p", "s"));
+
+ session.clearPortals();
+
+ assertSame(copy, session.statement("s").orElseThrow());
+ assertTrue(session.portal("p").isEmpty());
+ }
+
+ @Test
+ void clearRemovesStatementsAndPortals() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+ CopyStatementParser.S3Statement copy = CopyStatementParser.parse("COPY t FROM 's3://b/k'");
+ session.confirm(session.stageParse("s", copy));
+ session.confirm(session.stageBind("p", "s"));
+
+ session.clear();
+
+ assertTrue(session.statement("s").isEmpty());
+ assertTrue(session.portal("p").isEmpty());
+ }
+
+ @Test
+ void closeRejectsAnInvalidTargetType() {
+ ExtendedQuerySession session = new ExtendedQuerySession();
+
+ assertThrows(IllegalArgumentException.class, () -> session.stageClose('X', "name"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java
new file mode 100644
index 0000000000..468ab11255
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedS3ExchangeTest.java
@@ -0,0 +1,222 @@
+package io.github.hectorvent.floci.services.redshift.proxy;
+
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ExtendedS3ExchangeTest {
+
+ private Socket simClient;
+ private Socket simBackend;
+ private Socket testClient;
+ private Socket testBackend;
+ private S3Service s3;
+ private final AtomicReference<Throwable> backendFailure = new AtomicReference<>();
+
+ @BeforeEach
+ void setUp() throws IOException {
+ ServerSocket clientListener = new ServerSocket(0);
+ simClient = new Socket("localhost", clientListener.getLocalPort());
+ testClient = clientListener.accept();
+ clientListener.close();
+
+ ServerSocket backendListener = new ServerSocket(0);
+ simBackend = new Socket("localhost", backendListener.getLocalPort());
+ testBackend = backendListener.accept();
+ backendListener.close();
+ s3 = mock(S3Service.class);
+ }
+
+ @AfterEach
+ void tearDown() throws IOException {
+ for (Socket socket : new Socket[]{simClient, simBackend, testClient, testBackend}) {
+ if (socket != null && !socket.isClosed()) {
+ socket.close();
+ }
+ }
+ }
+
+ @Test
+ void copyInForwardsCommandCompleteWithoutReadyForQuery() throws Exception {
+ when(s3.objectExists("b", "in/data")).thenReturn(true);
+ when(s3.getObject("b", "in/data")).thenReturn(
+ new S3Object("b", "in/data", "1|a\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
+ CopyStatementParser.S3CopyFrom copy = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "in/data", "|", 0, false, false, null);
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());
+ BackendResponseCoordinator.Ticket ticket = coordinator.register(
+ BackendResponseCoordinator.Operation.EXECUTE, null);
+ ByteArrayOutputStream copied = new ByteArrayOutputStream();
+ writeSync();
+ Thread backend = backendThread(() -> playCopyIn(copied));
+
+ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, coordinator, ticket);
+ joinBackend(backend);
+
+ PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClient.getInputStream());
+ assertEquals('C', clientDecoder.nextMessage().type());
+ assertEquals("1|a\n", copied.toString(StandardCharsets.US_ASCII));
+ coordinator.onBackendFrame('Z', new byte[]{'I'});
+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));
+ testClient.setSoTimeout(200);
+ assertThrows(SocketTimeoutException.class, () -> testClient.getInputStream().read());
+ }
+
+ @Test
+ void copyOutWritesS3AndForwardsOnlyCommandComplete() throws Exception {
+ Map<String, byte[]> written = new ConcurrentHashMap<>();
+ when(s3.putObject(eq("b"), any(), any(), any(), any())).thenAnswer(invocation -> {
+ written.put(invocation.getArgument(1), invocation.getArgument(2));
+ return null;
+ });
+ CopyStatementParser.S3Unload unload = new CopyStatementParser.S3Unload(
+ "select a from t", "b", "out/", "|", false, false, false,
+ false, null, false, true, false, 0);
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());
+ BackendResponseCoordinator.Ticket ticket = coordinator.register(
+ BackendResponseCoordinator.Operation.EXECUTE, null);
+ writeSync();
+ Thread backend = backendThread(this::playCopyOut);
+
+ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), unload, s3, coordinator, ticket);
+ joinBackend(backend);
+
+ assertEquals("1\n2\n", new String(written.get("out/000"), StandardCharsets.US_ASCII));
+ PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClient.getInputStream());
+ assertEquals('C', clientDecoder.nextMessage().type());
+ coordinator.onBackendFrame('Z', new byte[]{'I'});
+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));
+ testClient.setSoTimeout(200);
+ assertThrows(SocketTimeoutException.class, () -> testClient.getInputStream().read());
+ }
+
+ @Test
+ void missingCopyInputSendsCopyFailAndWaitsForSyncRecovery() throws Exception {
+ CopyStatementParser.S3CopyFrom copy = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "missing", "|", 0, false, false, null);
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());
+ BackendResponseCoordinator.Ticket ticket = coordinator.register(
+ BackendResponseCoordinator.Operation.EXECUTE, null);
+ writeSync();
+ Thread backend = backendThread(() -> {
+ PostgresWireDecoder decoder = new PostgresWireDecoder(testBackend.getInputStream());
+ assertEquals('E', decoder.nextMessage().type());
+ testBackend.getOutputStream().write(new byte[]{'G', 0, 0, 0, 7, 0, 0, 0});
+ testBackend.getOutputStream().flush();
+ assertEquals('f', decoder.nextMessage().type());
+ assertEquals('S', decoder.nextMessage().type());
+ writeFrame(testBackend.getOutputStream(), 'E',
+ "SERROR\0C57014\0MCOPY aborted\0\0".getBytes(StandardCharsets.US_ASCII));
+ writeFrame(testBackend.getOutputStream(), 'Z', new byte[]{'E'});
+ testBackend.getOutputStream().flush();
+ });
+
+ ExtendedS3Exchange.execute(simClient, simBackend, executeFrame(), copy, s3, coordinator, ticket);
+ joinBackend(backend);
+
+ testClient.setSoTimeout(1_000);
+ PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClient.getInputStream());
+ PostgresWireDecoder.FrontendMessage error = clientDecoder.nextMessage();
+ assertEquals('E', error.type());
+ assertTrue(new String(error.body(), StandardCharsets.US_ASCII).contains("XX000"));
+ assertTrue(new String(error.body(), StandardCharsets.US_ASCII).contains("s3://b/missing"));
+ assertEquals('Z', clientDecoder.nextMessage().type());
+ assertEquals('E', coordinator.lastReadyStatus());
+ assertTrue(coordinator.awaitIdle(1, TimeUnit.SECONDS));
+ }
+
+ private static PostgresWireDecoder.FrontendMessage executeFrame() {
+ return new PostgresWireDecoder.FrontendMessage('E', new byte[]{0, 0, 0, 0, 0});
+ }
+
+ private void playCopyIn(ByteArrayOutputStream copied) throws IOException {
+ PostgresWireDecoder decoder = new PostgresWireDecoder(testBackend.getInputStream());
+ assertEquals('E', decoder.nextMessage().type());
+ testBackend.getOutputStream().write(new byte[]{'G', 0, 0, 0, 7, 0, 0, 0});
+ testBackend.getOutputStream().flush();
+ while (true) {
+ PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();
+ if (message.type() == 'd') {
+ copied.write(message.body());
+ } else if (message.type() == 'c') {
+ break;
+ }
+ }
+ assertEquals('S', decoder.nextMessage().type());
+ writeCommandComplete();
+ }
+
+ private void playCopyOut() throws IOException {
+ PostgresWireDecoder decoder = new PostgresWireDecoder(testBackend.getInputStream());
+ assertEquals('E', decoder.nextMessage().type());
+ OutputStream out = testBackend.getOutputStream();
+ out.write(new byte[]{'H', 0, 0, 0, 7, 0, 0, 0});
+ writeFrame(out, 'd', "1\n".getBytes(StandardCharsets.US_ASCII));
+ writeFrame(out, 'd', "2\n".getBytes(StandardCharsets.US_ASCII));
+ out.write(new byte[]{'c', 0, 0, 0, 4});
+ writeCommandComplete();
+ assertEquals('S', decoder.nextMessage().type());
+ }
+
+ private void writeCommandComplete() throws IOException {
+ writeFrame(testBackend.getOutputStream(), 'C', "COPY 2\0".getBytes(StandardCharsets.US_ASCII));
+ testBackend.getOutputStream().flush();
+ }
+
+ private void writeSync() throws IOException {
+ testClient.getOutputStream().write(new byte[]{'S', 0, 0, 0, 4});
+ testClient.getOutputStream().flush();
+ }
+
+ private static void writeFrame(OutputStream out, char type, byte[] body) throws IOException {
+ out.write(type);
+ out.write(new byte[]{0, 0, 0, (byte) (4 + body.length)});
+ out.write(body);
+ }
+
+ private Thread backendThread(BackendScript script) {
+ return Thread.ofVirtual().start(() -> {
+ try {
+ script.run();
+ } catch (Throwable failure) {
+ backendFailure.compareAndSet(null, failure);
+ }
+ });
+ }
+
+ private void joinBackend(Thread backend) throws InterruptedException {
+ backend.join();
+ Throwable failure = backendFailure.get();
+ if (failure != null) {
+ throw new AssertionError("fake backend failed", failure);
+ }
+ }
+
+ @FunctionalInterface
+ private interface BackendScript {
+ void run() throws IOException;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoderTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoderTest.java
index 3ebd828056..15669188d0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoderTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/PostgresWireDecoderTest.java
@@ -9,6 +9,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
+import java.util.List;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -20,6 +21,96 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
class PostgresWireDecoderTest {
+ @Test
+ void decodesParseNameSqlAndParameterTypeOids() throws IOException {
+ byte[] packet = frame('P', concat(
+ cString("stmt-1"),
+ cString("select $1::int"),
+ int16(1),
+ int32(23)));
+
+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(packet));
+ PostgresWireDecoder.ParseMessage parse = decoder.decodeParse(decoder.nextMessage());
+
+ assertEquals("stmt-1", parse.statementName());
+ assertEquals("select $1::int", parse.sql());
+ assertEquals(List.of(23), parse.parameterTypeOids());
+ }
+
+ @Test
+ void decodesBindNamesWithoutRetainingValueSemantics() throws IOException {
+ byte[] body = concat(cString("portal-1"), cString("stmt-1"), int16(0), int16(0), int16(0));
+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(frame('B', body)));
+
+ PostgresWireDecoder.BindMessage bind = decoder.decodeBind(decoder.nextMessage());
+
+ assertEquals("portal-1", bind.portalName());
+ assertEquals("stmt-1", bind.statementName());
+ }
+
+ @Test
+ void decodesExecuteDescribeAndClose() throws IOException {
+ PostgresWireDecoder.ExecuteMessage execute = decodeExecute(frame('E', concat(cString("p"), int32(0))));
+ PostgresWireDecoder.TargetMessage describe = decodeDescribe(frame('D', concat(new byte[]{'S'}, cString("s"))));
+ PostgresWireDecoder.TargetMessage close = decodeClose(frame('C', concat(new byte[]{'P'}, cString("p"))));
+
+ assertEquals(new PostgresWireDecoder.ExecuteMessage("p", 0), execute);
+ assertEquals(new PostgresWireDecoder.TargetMessage('S', "s"), describe);
+ assertEquals(new PostgresWireDecoder.TargetMessage('P', "p"), close);
+ }
+
+ @Test
+ void rejectsParseWithMissingCStringTerminator() {
+ PostgresWireDecoder decoder = decoderFor('P', "stmt-1".getBytes(StandardCharsets.UTF_8));
+ assertThrows(IOException.class, () -> decoder.decodeParse(decoder.nextMessage()));
+ }
+
+ @Test
+ void rejectsParseWithTruncatedInt16() {
+ byte[] body = concat(cString("stmt-1"), cString("select 1"), new byte[]{0});
+ PostgresWireDecoder decoder = decoderFor('P', body);
+ assertThrows(IOException.class, () -> decoder.decodeParse(decoder.nextMessage()));
+ }
+
+ @Test
+ void rejectsParseWithTruncatedOidList() {
+ byte[] body = concat(cString("stmt-1"), cString("select $1"), int16(1), new byte[]{0, 0});
+ PostgresWireDecoder decoder = decoderFor('P', body);
+ assertThrows(IOException.class, () -> decoder.decodeParse(decoder.nextMessage()));
+ }
+
+ @Test
+ void rejectsDescribeWithInvalidTargetByte() {
+ PostgresWireDecoder decoder = decoderFor('D', concat(new byte[]{'X'}, cString("name")));
+ assertThrows(IOException.class, () -> decoder.decodeDescribe(decoder.nextMessage()));
+ }
+
+ @Test
+ void rejectsCloseWithInvalidTargetByte() {
+ PostgresWireDecoder decoder = decoderFor('C', concat(new byte[]{'X'}, cString("name")));
+ assertThrows(IOException.class, () -> decoder.decodeClose(decoder.nextMessage()));
+ }
+
+ @Test
+ void rejectsExecuteWithTrailingBytes() {
+ PostgresWireDecoder decoder = decoderFor('E', concat(cString("p"), int32(0), new byte[]{1}));
+ assertThrows(IOException.class, () -> decoder.decodeExecute(decoder.nextMessage()));
+ }
+
+ @Test
+ void encodeParsePreservesNameAndTypeOidsWhileReplacingSql() throws IOException {
+ PostgresWireDecoder.ParseMessage original = new PostgresWireDecoder.ParseMessage(
+ "ddl", "CREATE TABLE t (id int ENCODE az64)", List.of(23, 25));
+
+ byte[] encoded = PostgresWireDecoder.encodeParse(original, "CREATE TABLE t (id int)");
+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(encoded));
+ PostgresWireDecoder.ParseMessage decoded = decoder.decodeParse(decoder.nextMessage());
+
+ assertEquals("ddl", decoded.statementName());
+ assertEquals("CREATE TABLE t (id int)", decoded.sql());
+ assertEquals(List.of(23, 25), decoded.parameterTypeOids());
+ }
+
@Test
void decodesASimpleQueryAndThenReportsEof() throws IOException {
byte[] packet = PostgresWireDecoder.encodeQuery("SELECT 1");
@@ -308,4 +399,69 @@ class PostgresWireDecoderTest {
assertNull(msg.body());
assertEquals(1 + (long) totalLength, bytesWritten[0]);
}
+
+ private static PostgresWireDecoder decoderFor(char type, byte[] body) {
+ return new PostgresWireDecoder(new ByteArrayInputStream(frame(type, body)));
+ }
+
+ private static PostgresWireDecoder.ExecuteMessage decodeExecute(byte[] packet) throws IOException {
+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(packet));
+ return decoder.decodeExecute(decoder.nextMessage());
+ }
+
+ private static PostgresWireDecoder.TargetMessage decodeDescribe(byte[] packet) throws IOException {
+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(packet));
+ return decoder.decodeDescribe(decoder.nextMessage());
+ }
+
+ private static PostgresWireDecoder.TargetMessage decodeClose(byte[] packet) throws IOException {
+ PostgresWireDecoder decoder = new PostgresWireDecoder(new ByteArrayInputStream(packet));
+ return decoder.decodeClose(decoder.nextMessage());
+ }
+
+ private static byte[] frame(char type, byte[] body) {
+ int length = body.length + 4;
+ byte[] packet = new byte[body.length + 5];
+ packet[0] = (byte) type;
+ packet[1] = (byte) ((length >> 24) & 0xFF);
+ packet[2] = (byte) ((length >> 16) & 0xFF);
+ packet[3] = (byte) ((length >> 8) & 0xFF);
+ packet[4] = (byte) (length & 0xFF);
+ System.arraycopy(body, 0, packet, 5, body.length);
+ return packet;
+ }
+
+ private static byte[] cString(String value) {
+ byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+ byte[] result = new byte[bytes.length + 1];
+ System.arraycopy(bytes, 0, result, 0, bytes.length);
+ return result;
+ }
+
+ private static byte[] int16(int value) {
+ return new byte[]{(byte) ((value >> 8) & 0xFF), (byte) (value & 0xFF)};
+ }
+
+ private static byte[] int32(int value) {
+ return new byte[]{
+ (byte) ((value >> 24) & 0xFF),
+ (byte) ((value >> 16) & 0xFF),
+ (byte) ((value >> 8) & 0xFF),
+ (byte) (value & 0xFF)
+ };
+ }
+
+ private static byte[] concat(byte[]... parts) {
+ int length = 0;
+ for (byte[] part : parts) {
+ length += part.length;
+ }
+ byte[] result = new byte[length];
+ int offset = 0;
+ for (byte[] part : parts) {
+ System.arraycopy(part, 0, result, offset, part.length);
+ offset += part.length;
+ }
+ return result;
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
index 2ab9e7b92e..2214d9476b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
@@ -5,8 +5,10 @@ import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.jboss.logging.Logger;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
import org.mockito.Mockito;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.ServerSocket;
@@ -123,6 +125,185 @@ class RedshiftInterceptingBridgeTest {
assertArrayEquals(parsePacket, nextForwarded().toPacketBytes());
}
+ @Test
+ void rewritesDdlParseWhilePreservingNameAndParameterOids() throws IOException {
+ startBridge();
+ PostgresWireDecoder.ParseMessage parse = new PostgresWireDecoder.ParseMessage(
+ "ddl", "CREATE TABLE sales (id int ENCODE az64, d date) "
+ + "DISTSTYLE KEY DISTKEY (id) SORTKEY (d)", List.of(23, 25));
+ testClientEnd.getOutputStream().write(PostgresWireDecoder.encodeParse(parse, parse.sql()));
+ testClientEnd.getOutputStream().flush();
+
+ PostgresWireDecoder backendDecoder = new PostgresWireDecoder(testBackendEnd.getInputStream());
+ PostgresWireDecoder.ParseMessage forwarded = backendDecoder.decodeParse(backendDecoder.nextMessage());
+
+ assertEquals("ddl", forwarded.statementName());
+ assertEquals(List.of(23, 25), forwarded.parameterTypeOids());
+ String sql = forwarded.sql().toUpperCase();
+ assertFalse(sql.contains("DISTSTYLE"), sql);
+ assertFalse(sql.contains("DISTKEY"), sql);
+ assertFalse(sql.contains("SORTKEY"), sql);
+ assertFalse(sql.contains("ENCODE"), sql);
+ }
+
+ @Test
+ void parameterizedCopyParseFallsOpenByteForByte() throws IOException {
+ startBridge();
+ PostgresWireDecoder.ParseMessage parse = new PostgresWireDecoder.ParseMessage(
+ "copy", "COPY t FROM 's3://b/in/data'", List.of(25));
+ byte[] packet = PostgresWireDecoder.encodeParse(parse, parse.sql());
+
+ testClientEnd.getOutputStream().write(packet);
+ testClientEnd.getOutputStream().flush();
+
+ assertArrayEquals(packet, nextForwarded().toPacketBytes());
+ }
+
+ @Test
+ void pipelinedExtendedCopyPreservesResponseOrderAndSyncReadyForQuery() throws Exception {
+ startBridge();
+ Mockito.when(s3Stub.objectExists("b", "in/data")).thenReturn(true);
+ Mockito.when(s3Stub.getObject("b", "in/data")).thenReturn(
+ new S3Object("b", "in/data", "1|a\n".getBytes(StandardCharsets.US_ASCII), "text/plain"));
+ AtomicReference<Throwable> backendFailure = new AtomicReference<>();
+ Thread backend = Thread.ofVirtual().start(() -> {
+ try {
+ PostgresWireDecoder decoder = new PostgresWireDecoder(testBackendEnd.getInputStream());
+ PostgresWireDecoder.ParseMessage parse = decoder.decodeParse(decoder.nextMessage());
+ assertTrue(parse.sql().contains("FROM STDIN"), parse.sql());
+ writeBackendFrame('1', new byte[0]);
+ assertEquals('B', decoder.nextMessage().type());
+ writeBackendFrame('2', new byte[0]);
+ assertEquals('D', decoder.nextMessage().type());
+ writeBackendFrame('n', new byte[0]);
+ assertEquals('E', decoder.nextMessage().type());
+ writeBackendFrame('G', new byte[]{0, 0, 0});
+ while (decoder.nextMessage().type() != 'c') {
+ // Consume CopyData until the bridge completes CopyIn.
+ }
+ assertEquals('S', decoder.nextMessage().type());
+ writeBackendFrame('C', "COPY 1\0".getBytes(StandardCharsets.US_ASCII));
+ writeBackendFrame('Z', new byte[]{'I'});
+ } catch (Throwable failure) {
+ backendFailure.compareAndSet(null, failure);
+ }
+ });
+
+ PostgresWireDecoder.ParseMessage parse = new PostgresWireDecoder.ParseMessage(
+ "copy-s", "COPY t FROM 's3://b/in/data'", List.of());
+ ByteArrayOutputStream pipeline = new ByteArrayOutputStream();
+ pipeline.write(PostgresWireDecoder.encodeParse(parse, parse.sql()));
+ pipeline.write(frame('B', concat(cString("copy-p"), cString("copy-s"), new byte[6])));
+ pipeline.write(frame('D', concat(new byte[]{'P'}, cString("copy-p"))));
+ pipeline.write(frame('E', concat(cString("copy-p"), new byte[4])));
+ pipeline.write(frame('S', new byte[0]));
+ testClientEnd.getOutputStream().write(pipeline.toByteArray());
+ testClientEnd.getOutputStream().flush();
+ PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClientEnd.getInputStream());
+ assertEquals('1', clientDecoder.nextMessage().type());
+ assertEquals('2', clientDecoder.nextMessage().type());
+ assertEquals('n', clientDecoder.nextMessage().type());
+ assertEquals('C', clientDecoder.nextMessage().type());
+ assertEquals('Z', clientDecoder.nextMessage().type());
+ backend.join();
+ if (backendFailure.get() != null) {
+ throw new AssertionError("fake backend failed", backendFailure.get());
+ }
+ }
+
+ @Test
+ @Timeout(5)
+ void extendedUnloadWithManifestCompletesAfterCopyOutCommandComplete() throws Exception {
+ startBridge();
+ Mockito.when(s3Stub.putObject(
+ Mockito.eq("b"), Mockito.anyString(), Mockito.any(), Mockito.anyString(), Mockito.any()))
+ .thenReturn(null);
+ AtomicReference<Throwable> backendFailure = new AtomicReference<>();
+ Thread backend = Thread.ofVirtual().start(() -> {
+ try {
+ PostgresWireDecoder decoder = new PostgresWireDecoder(testBackendEnd.getInputStream());
+ PostgresWireDecoder.ParseMessage parse = decoder.decodeParse(decoder.nextMessage());
+ assertTrue(parse.sql().startsWith("COPY (select id from t) TO STDOUT"), parse.sql());
+ assertEquals('B', decoder.nextMessage().type());
+ assertEquals('D', decoder.nextMessage().type());
+ assertEquals('E', decoder.nextMessage().type());
+ assertEquals('S', decoder.nextMessage().type());
+ writeBackendFrame('1', new byte[0]);
+ writeBackendFrame('2', new byte[0]);
+ writeBackendFrame('n', new byte[0]);
+ writeBackendFrame('H', new byte[]{0, 0, 0});
+ writeBackendFrame('d', "1\n".getBytes(StandardCharsets.US_ASCII));
+ writeBackendFrame('c', new byte[0]);
+ writeBackendFrame('C', "COPY 1\0".getBytes(StandardCharsets.US_ASCII));
+ writeBackendFrame('Z', new byte[]{'I'});
+ } catch (Throwable failure) {
+ backendFailure.compareAndSet(null, failure);
+ }
+ });
+
+ PostgresWireDecoder.ParseMessage parse = new PostgresWireDecoder.ParseMessage(
+ "unload-s", "UNLOAD ('select id from t') TO 's3://b/out/' MANIFEST", List.of());
+ ByteArrayOutputStream pipeline = new ByteArrayOutputStream();
+ pipeline.write(PostgresWireDecoder.encodeParse(parse, parse.sql()));
+ pipeline.write(frame('B', concat(cString("unload-p"), cString("unload-s"), new byte[6])));
+ pipeline.write(frame('D', concat(new byte[]{'P'}, cString("unload-p"))));
+ pipeline.write(frame('E', concat(cString("unload-p"), new byte[4])));
+ pipeline.write(frame('S', new byte[0]));
+ testClientEnd.getOutputStream().write(pipeline.toByteArray());
+ testClientEnd.getOutputStream().flush();
+ testClientEnd.setSoTimeout(3000);
+
+ backend.join(1000);
+ if (backendFailure.get() != null) {
+ throw new AssertionError("fake backend failed", backendFailure.get());
+ }
+ PostgresWireDecoder clientDecoder = new PostgresWireDecoder(testClientEnd.getInputStream());
+ assertEquals('1', clientDecoder.nextMessage().type());
+ assertEquals('2', clientDecoder.nextMessage().type());
+ assertEquals('n', clientDecoder.nextMessage().type());
+ assertEquals('C', clientDecoder.nextMessage().type());
+ assertEquals('Z', clientDecoder.nextMessage().type());
+ backend.join();
+ if (backendFailure.get() != null) {
+ throw new AssertionError("fake backend failed", backendFailure.get());
+ }
+ }
+
+ private void writeBackendFrame(char type, byte[] body) throws IOException {
+ testBackendEnd.getOutputStream().write(frame(type, body));
+ testBackendEnd.getOutputStream().flush();
+ }
+
+ private static byte[] frame(char type, byte[] body) {
+ int length = 4 + body.length;
+ byte[] packet = new byte[5 + body.length];
+ packet[0] = (byte) type;
+ packet[1] = (byte) (length >>> 24);
+ packet[2] = (byte) (length >>> 16);
+ packet[3] = (byte) (length >>> 8);
+ packet[4] = (byte) length;
+ System.arraycopy(body, 0, packet, 5, body.length);
+ return packet;
+ }
+
+ private static byte[] cString(String value) {
+ return concat(value.getBytes(StandardCharsets.UTF_8), new byte[]{0});
+ }
+
+ private static byte[] concat(byte[]... values) {
+ int length = 0;
+ for (byte[] value : values) {
+ length += value.length;
+ }
+ byte[] result = new byte[length];
+ int offset = 0;
+ for (byte[] value : values) {
+ System.arraycopy(value, 0, result, offset, value.length);
+ offset += value.length;
+ }
+ return result;
+ }
+
@Test
void pumpsBackendBytesToTheClientUnchanged() throws IOException {
startBridge();
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java
index 9570a4158d..6b19bf101b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/S3CopySimulatorTest.java
@@ -143,6 +143,58 @@ class S3CopySimulatorTest {
"|", false, gzip, false, false, null, manifest, allowOverwrite, parallel, maxFileSize);
}
+ @Test
+ void preparedCopyUsesSortedKeysAndBackendSql() {
+ when(s3.objectExists("b", "prefix/")).thenReturn(false);
+ when(s3.listObjectsWithPrefixes(
+ eq("b"), eq("prefix/"), isNull(), anyInt(), isNull(), isNull()))
+ .thenReturn(new S3Service.ListObjectsResult(
+ List.of(
+ new S3Object("b", "prefix/z", new byte[0], "text/plain"),
+ new S3Object("b", "prefix/a", new byte[0], "text/plain")),
+ List.of(), false, null));
+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "prefix/", null, 0, false, true, null);
+
+ S3CopySimulator.CopyInput input = S3CopySimulator.prepareCopy(spec, s3);
+
+ assertEquals(List.of("prefix/a", "prefix/z"), input.keys());
+ assertEquals("COPY t FROM STDIN WITH (FORMAT csv, DELIMITER ',')",
+ S3CopySimulator.copyBackendSql(spec));
+ }
+
+ @Test
+ void preparedCopyReportsMissingObject() {
+ CopyStatementParser.S3CopyFrom spec = new CopyStatementParser.S3CopyFrom(
+ "t", List.of(), "b", "missing", "|", 0, false, false, null);
+
+ S3CopySimulator.S3TransferException error = assertThrows(
+ S3CopySimulator.S3TransferException.class,
+ () -> S3CopySimulator.prepareCopy(spec, s3));
+
+ assertEquals("XX000", error.sqlState());
+ assertEquals("S3 object s3://b/missing not found", error.getMessage());
+ }
+
+ @Test
+ void unloadCollectorWritesOneEmptyObjectForZeroRows() throws Exception {
+ Map<String, byte[]> written = new ConcurrentHashMap<>();
+ when(s3.putObject(eq("b"), any(), any(), any(), any())).thenAnswer(invocation -> {
+ written.put(invocation.getArgument(1), invocation.getArgument(2));
+ return null;
+ });
+ CopyStatementParser.S3Unload spec = unloadSpec("b", "out/", false, false, true, false, 0);
+
+ try (S3CopySimulator.UnloadCollector collector = S3CopySimulator.prepareUnload(spec, s3)) {
+ collector.complete();
+ }
+
+ assertEquals(1, written.size());
+ assertEquals(0, written.get("out/000").length);
+ assertEquals("COPY (select a,b from t) TO STDOUT WITH (FORMAT text, DELIMITER '|')",
+ S3CopySimulator.unloadBackendSql(spec));
+ }
+
@Test
void unloadWritesSingleObjectAndForwardsCommandComplete() throws Exception {
Map<String, byte[]> written = new ConcurrentHashMap<>();