Light, fluffy, and always free - The AWS Local Emulator alternative
feat(redshift): add relational queries for Redshift Spectrum (#5095)
Vũ Thế Huy950b6fe774cc
diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftRelationalSpectrumTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftRelationalSpectrumTest.javanew file mode 100644index 0000000000..ba459377c1--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftRelationalSpectrumTest.java@@ -0,0 +1,143 @@+package io.github.hectorvent.floci.compat;++import com.floci.test.TestFixtures;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.core.sync.RequestBody;+import software.amazon.awssdk.services.glue.GlueClient;+import software.amazon.awssdk.services.glue.model.DatabaseInput;+import software.amazon.awssdk.services.iam.IamClient;+import software.amazon.awssdk.services.redshift.RedshiftClient;+import software.amazon.awssdk.services.redshift.model.Cluster;+import software.amazon.awssdk.services.redshiftdata.RedshiftDataClient;+import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementResponse;+import software.amazon.awssdk.services.redshiftdata.model.GetStatementResultResponse;+import software.amazon.awssdk.services.s3.S3Client;++import java.sql.Connection;+import java.sql.DriverManager;+import java.sql.PreparedStatement;+import java.sql.ResultSet;+import java.sql.Statement;+import java.time.Instant;+import java.util.UUID;++import static org.junit.jupiter.api.Assertions.*;++class RedshiftRelationalSpectrumTest {+ @Test+ @DisplayName("Glue external tables join internal tables through JDBC and the SDK Data API")+ void joinsGlueCsvTablesThroughJdbcAndDataApi() throws Exception {+ String suffix = UUID.randomUUID().toString().replace("-", "").substring(0, 12);+ String clusterId = "spectrum-rel-" + suffix;+ String bucket = "spectrum-rel-" + suffix;+ String database = "spectrum_rel_" + suffix;+ String roleName = "SpectrumRel" + suffix;+ boolean clusterCreated = false;+ boolean bucketCreated = false;+ boolean databaseCreated = false;+ boolean roleCreated = false;+ boolean policyCreated = false;+ try (RedshiftClient redshift = TestFixtures.redshiftClient();+ RedshiftDataClient data = TestFixtures.redshiftDataClient();+ GlueClient glue = TestFixtures.glueClient();+ S3Client s3 = TestFixtures.s3Client();+ IamClient iam = TestFixtures.iamClient()) {+ try {+ String roleArn = iam.createRole(r -> r.roleName(roleName).assumeRolePolicyDocument("""+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",+ "Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}+ """)).role().arn();+ roleCreated = true;+ iam.putRolePolicy(r -> r.roleName(roleName).policyName("SpectrumRead").policyDocument("""+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",+ "Action":["s3:GetObject","s3:ListBucket","glue:GetTable","glue:GetPartitions"],"Resource":"*"}]}+ """));+ policyCreated = true;+ s3.createBucket(r -> r.bucket(bucket));+ bucketCreated = true;+ s3.putObject(r -> r.bucket(bucket).key("events/data.csv"), RequestBody.fromString("1,10\n1,5\n2,7\n"));+ glue.createDatabase(r -> r.databaseInput(DatabaseInput.builder().name(database).build()));+ databaseCreated = true;+ redshift.createCluster(r -> r.clusterIdentifier(clusterId).nodeType("dc2.large")+ .masterUsername("admin").masterUserPassword("Password123").iamRoles(roleArn));+ clusterCreated = true;+ Cluster cluster = redshift.describeClusters(r -> r.clusterIdentifier(clusterId)).clusters().get(0);+ String url = "jdbc:postgresql://" + cluster.endpoint().address() + ":" + cluster.endpoint().port() + "/dev";+ try (Connection connection = DriverManager.getConnection(url, "admin", "Password123");+ Statement setup = connection.createStatement()) {+ setup.execute("CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE '" + database+ + "' IAM_ROLE '" + roleArn + "'");+ setup.execute("CREATE EXTERNAL TABLE lake.events (id INTEGER, amount INTEGER) "+ + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE LOCATION 's3://"+ + bucket + "/events/'");+ setup.execute("CREATE EXTERNAL TABLE lake.mirror_events (id INTEGER, amount INTEGER) "+ + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE LOCATION 's3://"+ + bucket + "/events/'");+ assertEquals("events", glue.getTable(r -> r.databaseName(database).name("events")).table().name());+ setup.execute("CREATE TABLE customers(id INTEGER, country VARCHAR(2))");+ setup.execute("INSERT INTO customers VALUES (1,'VN'),(2,'US')");+ String sql = "SELECT c.country, SUM(e.amount) AS total FROM lake.events e "+ + "JOIN (SELECT DISTINCT id FROM lake.mirror_events) m ON m.id=e.id "+ + "JOIN customers c ON c.id=e.id GROUP BY c.country ORDER BY total DESC";+ try (PreparedStatement query = connection.prepareStatement(sql);+ ResultSet rows = query.executeQuery()) {+ assertTrue(rows.next());+ assertEquals("VN", rows.getString(1));+ assertEquals(15L, rows.getLong(2));+ assertTrue(rows.next());+ assertEquals("US", rows.getString(1));+ assertEquals(7L, rows.getLong(2));+ assertFalse(rows.next());+ }+ String id = data.executeStatement(r -> r.clusterIdentifier(clusterId).database("dev")+ .dbUser("admin").sql(sql)).id();+ Instant deadline = Instant.now().plusSeconds(30);+ DescribeStatementResponse status = data.describeStatement(r -> r.id(id));+ while (!status.statusAsString().equals("FINISHED") && Instant.now().isBefore(deadline)) {+ assertNotEquals("FAILED", status.statusAsString(), status.error());+ Thread.sleep(100);+ status = data.describeStatement(r -> r.id(id));+ }+ assertEquals("FINISHED", status.statusAsString(), status.error());+ GetStatementResultResponse result = data.getStatementResult(r -> r.id(id));+ assertEquals(2L, result.totalNumRows());+ assertEquals("VN", result.records().get(0).get(0).stringValue());+ assertEquals(15L, result.records().get(0).get(1).longValue());+ assertEquals("US", result.records().get(1).get(0).stringValue());+ String failedBatch = data.batchExecuteStatement(r -> r.clusterIdentifier(clusterId).database("dev")+ .dbUser("admin").sqls("INSERT INTO customers VALUES (3,'JP')", "SELECT * FROM lake.absent")).id();+ Instant failureDeadline = Instant.now().plusSeconds(30);+ DescribeStatementResponse failed = data.describeStatement(r -> r.id(failedBatch));+ while (!failed.statusAsString().equals("FAILED") && Instant.now().isBefore(failureDeadline)) {+ assertNotEquals("FINISHED", failed.statusAsString());+ Thread.sleep(100);+ failed = data.describeStatement(r -> r.id(failedBatch));+ }+ assertEquals("FAILED", failed.statusAsString());+ try (ResultSet rows = setup.executeQuery("SELECT COUNT(*) FROM customers WHERE id=3")) {+ assertTrue(rows.next());+ assertEquals(0L, rows.getLong(1));+ }+ }+ } finally {+ if (clusterCreated) {+ redshift.deleteCluster(r -> r.clusterIdentifier(clusterId).skipFinalClusterSnapshot(true));+ }+ if (databaseCreated) {+ glue.deleteDatabase(r -> r.name(database));+ }+ if (bucketCreated) {+ s3.deleteObject(r -> r.bucket(bucket).key("events/data.csv"));+ s3.deleteBucket(r -> r.bucket(bucket));+ }+ if (policyCreated) {+ iam.deleteRolePolicy(r -> r.roleName(roleName).policyName("SpectrumRead"));+ }+ if (roleCreated) {+ iam.deleteRole(r -> r.roleName(roleName));+ }+ }+ }+ }+}diff --git a/docs/services/redshift-data.md b/docs/services/redshift-data.mdindex 49b2d40f86..0c437fb21a 100644--- a/docs/services/redshift-data.md+++ b/docs/services/redshift-data.md@@ -58,6 +58,20 @@ A statement run against a workgroup reports `WorkgroupName` and omits `ClusterId- **`ExecuteSql` and `BatchExecuteSql`** (the deprecated pre-2020 operations) return `ValidationException`.- **Type mapping.** JDBC `BOOLEAN` and `BIT` map to `booleanValue`; integer types to `longValue`; floating-point types to `doubleValue`; `NUMERIC` and `DECIMAL` to `stringValue` (as AWS does); binary types to `blobValue`; everything else, including dates, timestamps, and uuid, to `stringValue`. A SQL `NULL` maps to `isNull`. A result column of type `line`, `json`, or `jsonb` fails the statement with the Redshift error text.+## Spectrum relational queries++The Data API prepares Glue-backed external CSV tables on the same JDBC connection that executes+the SQL. JOINs with internal tables, multiple external tables, grouping, ordering and parameters+follow the [Redshift Spectrum relational path](redshift.md#relational-queries-over-glue-backed-external-csv-tables).+Create the external schema through either the wire endpoint or the Data API; both use the same+runtime and database binding. Batch preparation runs inside the existing batch transaction, so+failure rolls back the batch. External read or preparation failures are stored as `FAILED` and+reported by `DescribeStatement`, as other SQL execution failures are.+Create external schemas and tables outside a batch: `CREATE EXTERNAL TABLE` cannot run inside a transaction block on AWS, and a batch runs as one transaction, so both statements are rejected there.++This does not add Data API session reuse or asynchronous execution. The legacy Phase 1 catalog+is not the Glue-backed relational path.+## Configuration| Variable | Default | Description |diff --git a/docs/services/redshift.md b/docs/services/redshift.mdindex b1508e23c3..1a961ba134 100644--- a/docs/services/redshift.md+++ b/docs/services/redshift.md@@ -272,6 +272,63 @@ current S3 emulator authorization mode, and role-specific Spectrum access enforcfuture phase. Parquet, JSON, Avro, ORC, partition discovery, `ALTER`, and `DROP` lifecycleoperations are not supported in Phase 1.+### Relational queries over Glue-backed external CSV tables++An external schema bound to an existing local Glue database supports JOINs between external and+internal tables, multiple external tables, aliases, bound predicates, GROUP BY, HAVING, ORDER BY+and LIMIT. PostgreSQL evaluates the original SQL after Floci loads the referenced external+relations. Simple Query, JDBC Extended Query and the Redshift Data API share this preparation path.+Extended Query uses native PostgreSQL descriptions and cursors, including fetch-size cursors.++Create the Glue database first, or use `CREATE EXTERNAL DATABASE IF NOT EXISTS`:++```sql+CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG+DATABASE 'analytics' IAM_ROLE '<associated-role-arn>'+CREATE EXTERNAL DATABASE IF NOT EXISTS;++CREATE EXTERNAL TABLE lake.events (customer_id INTEGER, amount INTEGER)+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','+STORED AS TEXTFILE LOCATION 's3://analytics/events/';++SELECT c.country, SUM(e.amount) AS total+FROM lake.events e JOIN public.customers c ON e.customer_id = c.id+GROUP BY c.country ORDER BY total DESC;+```++This path needs Docker for PostgreSQL and the existing floci-duck runtime used to read external+data. Use the `floci.duck` configuration described under [Athena](athena.md) to supply a reachable+runtime when container management is unsuitable. This iteration validates CSV relational reads;+it does not add new file formats or partition pruning.++Materialized relations are cached in the cluster database. Each new query checks the associated+role, trust policy and S3 access before using cached data, following the existing S3 enforcement+configuration. Changes to Glue metadata or S3 objects reload the data. A missing Glue table is an+error even if its previous contents remain cached. New prepared-statement bindings revalidate+the data; repeated fetches from one active cursor retain that cursor's result. Loads inside a+caller transaction do not publish a reusable fingerprint because the transaction may roll back.+Temporary transfer objects and staging tables are cleaned up independently of cached relations.+Extended Query loads retain a connection-local fingerprint until its transaction completes;+the fingerprint becomes shared only after confirmed commit and is discarded on rollback.+Creating an external schema or table+inside a transaction, including a Data API batch (which runs as one transaction), is rejected before changing the catalogs: AWS does not allow `CREATE EXTERNAL TABLE` inside a transaction block (SQLSTATE `25001`).+Cold loads and reloads require database privileges to maintain the materialized table and catalog;+ordinary users with schema USAGE and table SELECT can read an existing committed cache entry.+The current floci-duck transfer uses account-scoped development credentials. Cold loads and reloads+with `FLOCI_SERVICES_S3_ENFORCE_AUTH=true` are not supported by that transfer path yet.+Role association and trust are still checked before cached reads; S3 policy checks follow the+configured enforcement mode. This is a Floci limitation, not AWS behavior.++Writes to external relations are rejected. AWS prohibits external UPDATE/DELETE but supports+some external INSERT operations; external INSERT is an intentional Floci limitation here. Use+schema-qualified external table names. Full Glue IAM enforcement, Lake Formation, role chaining+and `IAM_ROLE SESSION` are outside this implementation. Metadata is exposed through+`svv_external_schemas`, `svv_external_tables`, `svv_external_columns` and `svv_external_partitions`.++Schemas created without a matching Glue database retain the legacy Phase 1 path described above.+Its single-table query restrictions still apply. PostgreSQL materialization approximates Spectrum+execution and does not reproduce AWS distributed planning or a cross-service transaction snapshot.+### COPY from S3`COPY <table> [(<columns>)] FROM 's3://<bucket>/<keyOrPrefix>' [options]` sent over the Simplediff --git a/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckManager.java b/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckManager.javaindex db37bfe137..e9a62b5fa8 100644--- a/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckManager.java@@ -122,6 +122,7 @@ public class FlociDuckManager {.withDynamicPort(DUCK_PORT).withDockerNetwork(config.services().dockerNetwork()).withEmbeddedDns()+ .withHostDockerInternalOnLinux().withLogRotation().build();diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.javaindex 798e321d93..869200abd1 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java@@ -603,6 +603,7 @@ public class RedshiftService {}containerManager.stop(clusters.accountId(), identifier);+ proxyManager.forgetSpectrumRuntime(relayKey(clusters.accountId(), identifier));clusters.delete(identifier);clusters.flush();// Invalidate any GetClusterCredentials passwords so a cluster later recreated with thisdiff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.javaindex 71ec88f443..c89d3903c8 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.redshift.proxy;+import java.nio.charset.StandardCharsets;import java.util.ArrayDeque;import java.util.HashMap;import java.util.Map;@@ -106,6 +107,9 @@ final class BackendResponseCoordinator {void onBackendFrame(char type, byte[] body) {lock.lock();try {+ if (type == 'C') {+ updateStatusFromCommandTag(new String(body, StandardCharsets.UTF_8));+ }if (type == 'Z') {updateReadyStatus(body);}@@ -215,6 +219,28 @@ final class BackendResponseCoordinator {}}+ /**+ * Tracks transaction boundaries as the tag arrives, so a statement pipelined after COMMIT sees them. The+ * ROLLBACK tag is also sent for ROLLBACK TO SAVEPOINT, which leaves the transaction open, so only the+ * ReadyForQuery status can say that a rollback ended it.+ */+ private void updateStatusFromCommandTag(String tag) {+ boolean transactionEnded = switch (tag) {+ case "BEGIN\0" -> {+ lastReadyStatus = 'T';+ yield false;+ }+ case "COMMIT\0", "END\0" -> {+ lastReadyStatus = 'I';+ yield true;+ }+ default -> false;+ };+ if (transactionEnded) {+ session.transactionEnded();+ }+ }+private void updateReadyStatus(byte[] body) {if (body != null && body.length == 1) {lastReadyStatus = (char) body[0];diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.javaindex ff323c6d1d..57381b0619 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java@@ -5,8 +5,8 @@ import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;import io.github.hectorvent.floci.services.rds.proxy.PostgresProtocolHandler;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;-import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumInterceptor;+import io.github.hectorvent.floci.services.s3.S3Service;import org.jboss.logging.Logger;import java.io.IOException;@@ -222,7 +222,7 @@ public class RedshiftAuthProxy {// Redshift-only DDL (DISTKEY/SORTKEY/ENCODE/...) is rewritten for the plain// PostgreSQL backend on the way through; every other message is relayed verbatim.new RedshiftInterceptingBridge(session.client(), session.backend(), s3Service, iamService,- clusterAccountId, iamRoleArns, spectrumInterceptor).run();+ clusterAccountId, iamRoleArns, spectrumInterceptor, clusterKey, dbName).run();}} catch (Exception e) {LOG.debugv("Redshift connection error for cluster {0}: {1}", clusterKey, e.getMessage());diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.javaindex bfd1085f85..6615d95845 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java@@ -1,10 +1,12 @@package io.github.hectorvent.floci.services.redshift.proxy;import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.services.redshift.spectrum.PostgresExtendedBackendSession;import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumColumn;import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumInterceptor;import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumQueryRewriter;import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumReadException;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSession;import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSqlException;import io.github.hectorvent.floci.services.s3.S3Service;import org.jboss.logging.Logger;@@ -60,6 +62,10 @@ public class RedshiftInterceptingBridge {private final String clusterAccountId;private final List<String> iamRoleArns;private final SpectrumInterceptor spectrumInterceptor;+ private final String clusterKey;+ private final String databaseName;+ private final PostgresExtendedBackendSession relationalBackend;+ private final Map<String, PostgresWireDecoder.ParseMessage> relationalStatements = new HashMap<>();private final ExtendedQuerySession session = new ExtendedQuerySession();private final Map<String, SpectrumInterceptor.Plan> spectrumStatements = new HashMap<>();private final Map<String, SpectrumInterceptor.Plan> spectrumPortals = new HashMap<>();@@ -70,6 +76,7 @@ public class RedshiftInterceptingBridge {private volatile boolean pumpBetweenMessages = true;private volatile boolean pumpFinished = false;private boolean extendedSpectrumError;+ private boolean rolledBackInCycle;public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService) {this(client, backend, s3Service, iamService, null);@@ -82,12 +89,13 @@ public class RedshiftInterceptingBridge {public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService,String clusterAccountId, List<String> iamRoleArns) {- this(client, backend, s3Service, iamService, clusterAccountId, iamRoleArns, null);+ // Without a Spectrum interceptor the cluster key and database name are never read.+ this(client, backend, s3Service, iamService, clusterAccountId, iamRoleArns, null, null, null);}public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService,String clusterAccountId, List<String> iamRoleArns,- SpectrumInterceptor spectrumInterceptor) {+ SpectrumInterceptor spectrumInterceptor, String clusterKey, String databaseName) {this.client = client;this.backend = backend;this.s3Service = s3Service;@@ -95,6 +103,14 @@ public class RedshiftInterceptingBridge {this.clusterAccountId = clusterAccountId;this.iamRoleArns = iamRoleArns == null ? List.of() : List.copyOf(iamRoleArns);this.spectrumInterceptor = spectrumInterceptor;+ this.clusterKey = clusterKey;+ this.databaseName = databaseName;+ this.relationalBackend = new PostgresExtendedBackendSession(backend);+ }++ private SpectrumSession spectrumSession() {+ return new SpectrumSession(clusterAccountId, clusterKey, databaseName, iamRoleArns,+ coordinator.lastReadyStatus() != 'I');}@FunctionalInterface@@ -171,6 +187,9 @@ public class RedshiftInterceptingBridge {} catch (Exception e) {LOG.warnv(e, "Unexpected error in RedshiftInterceptingBridge");} finally {+ if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null) {+ spectrumInterceptor.preparation().finishCycle(relationalBackend);+ }coordinator.close();closeQuietly(client, "client");closeQuietly(backend, "backend");@@ -185,18 +204,22 @@ public class RedshiftInterceptingBridge {boolean intercepted;try {intercepted = runWithBackendOwned(() -> {- decision[0] = spectrumInterceptor.intercept(sql, clusterAccountId, "dev", backend);+ decision[0] = spectrumInterceptor.intercept(sql, spectrumSession(), backend);return true;});- } catch (SpectrumSqlException | SpectrumReadException | IllegalArgumentException exception) {+ } catch (RuntimeException exception) {writeSimpleSpectrumError(exception);return;}+ if (!intercepted && spectrumInterceptor.preparation() != null+ && spectrumInterceptor.preparation().referencesExternal(sql, spectrumSession())) {+ writeSimpleSpectrumError(new SpectrumReadException("58030", "Unable to prepare external query"));+ return;+ }if (intercepted && decision[0] instanceof SpectrumInterceptor.Decision.Handled) {coordinator.register(BackendResponseCoordinator.Operation.SIMPLE_QUERY, null);write(client.getOutputStream(), commandComplete("CREATE EXTERNAL\0"));- write(client.getOutputStream(), readyForQuery('I'));- coordinator.onBackendFrame('Z', new byte[]{'I'});+ reportLocalReady();return;}if (intercepted && decision[0] instanceof SpectrumInterceptor.Decision.Rewritten rewritten) {@@ -246,16 +269,39 @@ public class RedshiftInterceptingBridge {// it, per protocol; without this, a later non-Spectrum statement reusing that name would still// resolve to the stale Spectrum plan in handleBind/handleDescribe/handleExecute.spectrumStatements.remove(parse.statementName());+ relationalStatements.remove(parse.statementName());+ // After a backend error the backend discards everything up to Sync, so a preparation round trip+ // would only wait out the socket timeout; forward the Parse and let the backend discard it.+ if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null+ && !coordinator.isDiscardingUntilSync()+ && spectrumInterceptor.preparation().referencesExternal(parse.sql(), spectrumSession())) {+ try {+ write(backendOut, new byte[]{'H', 0, 0, 0, 4});+ boolean prepared = runWithBackendOwned(() -> {+ spectrumInterceptor.preparation().prepare(parse.sql(), spectrumSession(), relationalBackend);+ return true;+ });+ if (!prepared) {+ throw new SpectrumReadException("58030", "Unable to prepare external query");+ }+ relationalStatements.put(parse.statementName(), parse);+ } catch (RuntimeException exception) {+ spectrumInterceptor.preparation().finishCycle(relationalBackend);+ writeParseSpectrumError(exception);+ return;+ }+ }if (spectrumInterceptor != null && parse.parameterTypeOids().isEmpty()) {try {- SpectrumInterceptor.Plan plan = spectrumInterceptor.plan(parse.sql(), clusterAccountId, "dev");- if (plan instanceof SpectrumInterceptor.Plan.Ddl || plan instanceof SpectrumInterceptor.Plan.Query) {+ SpectrumInterceptor.Plan plan = spectrumInterceptor.plan(parse.sql(), spectrumSession());+ if (plan instanceof SpectrumInterceptor.Plan.Ddl || plan instanceof SpectrumInterceptor.Plan.Query+ || plan instanceof SpectrumInterceptor.Plan.ExternalDdl) {awaitPriorBackendResponses();spectrumStatements.put(parse.statementName(), plan);write(client.getOutputStream(), backendFrame('1', EMPTY_BODY));return;}- } catch (SpectrumSqlException | SpectrumReadException | IllegalArgumentException exception) {+ } catch (RuntimeException exception) {writeParseSpectrumError(exception);return;}@@ -285,6 +331,24 @@ public class RedshiftInterceptingBridge {private void handleBind(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,OutputStream backendOut) throws IOException {PostgresWireDecoder.BindMessage bind = decoder.decodeBind(message);+ PostgresWireDecoder.ParseMessage relational = relationalStatements.get(bind.statementName());+ if (relational != null && !coordinator.isDiscardingUntilSync()) {+ try {+ // Parse/Describe responses may be buffered until Flush or Sync; drain them before taking ownership.+ write(backendOut, new byte[]{'H', 0, 0, 0, 4});+ boolean prepared = runWithBackendOwned(() -> {+ spectrumInterceptor.preparation().prepare(relational.sql(), spectrumSession(), relationalBackend);+ return true;+ });+ if (!prepared) {+ throw new SpectrumReadException("58030", "Unable to prepare external query");+ }+ } catch (RuntimeException exception) {+ spectrumInterceptor.preparation().finishCycle(relationalBackend);+ writeParseSpectrumError(exception);+ return;+ }+ }SpectrumInterceptor.Plan spectrumPlan = spectrumStatements.get(bind.statementName());if (spectrumPlan != null) {awaitPriorBackendResponses();@@ -315,7 +379,7 @@ public class RedshiftInterceptingBridge {}byte[] response = spectrumDescribeResponse(spectrumPlan);write(client.getOutputStream(), response);- } catch (SpectrumSqlException | SpectrumReadException | IllegalArgumentException exception) {+ } catch (RuntimeException exception) {writeParseSpectrumError(exception);}return;@@ -425,7 +489,9 @@ public class RedshiftInterceptingBridge {}try {backend.setSoTimeout(EXCHANGE_READ_TIMEOUT_MS);- ExtendedSpectrumExchange.execute(client, backend, plan, spectrumInterceptor, clusterAccountId, "dev",+ SpectrumInterceptor.Plan executionPlan = plan instanceof SpectrumInterceptor.Plan.ExternalDdl ddl+ ? new SpectrumInterceptor.Plan.ExternalDdl(ddl.sql(), spectrumSession()) : plan;+ ExtendedSpectrumExchange.execute(client, backend, executionPlan, spectrumInterceptor, clusterAccountId, databaseName,coordinator, ticket, portalName, maxRows, spectrumCursors);} finally {try {@@ -442,6 +508,7 @@ public class RedshiftInterceptingBridge {PostgresWireDecoder.TargetMessage close = decoder.decodeClose(message);boolean spectrumOwned;if (close.targetType() == 'S') {+ relationalStatements.remove(close.name());SpectrumInterceptor.Plan closedPlan = spectrumStatements.remove(close.name());spectrumOwned = closedPlan != null;if (closedPlan != null) {@@ -602,8 +669,14 @@ public class RedshiftInterceptingBridge {private void writeSimpleSpectrumError(RuntimeException exception) throws IOException {coordinator.register(BackendResponseCoordinator.Operation.SIMPLE_QUERY, null);write(client.getOutputStream(), errorResponse(spectrumSqlState(exception), exception.getMessage()));- write(client.getOutputStream(), readyForQuery('I'));- coordinator.onBackendFrame('Z', new byte[]{'I'});+ reportLocalReady();+ }++ /** A query answered locally never reaches the backend, so its transaction state must be reported unchanged. */+ private void reportLocalReady() throws IOException {+ char status = coordinator.lastReadyStatus();+ write(client.getOutputStream(), readyForQuery(status));+ coordinator.onBackendFrame('Z', new byte[]{(byte) status});}private void writeParseSpectrumError(RuntimeException exception) throws IOException {@@ -643,7 +716,11 @@ public class RedshiftInterceptingBridge {if (exception instanceof SpectrumReadException spectrumReadException) {return spectrumReadException.sqlState();}- return "22023";+ if (exception instanceof IllegalArgumentException) {+ return "22023";+ }+ LOG.warnv(exception, "External query preparation failed unexpectedly");+ return "58030";}/**@@ -710,7 +787,38 @@ public class RedshiftInterceptingBridge {}private void pumpBackendToClient() {- WireFrameTracker tracker = new WireFrameTracker(coordinator::onBackendFrame);+ WireFrameTracker tracker = new WireFrameTracker((type, body) -> {+ // Publish or drop pending preparation state before the coordinator reports the backend idle,+ // so the next preparation already sees a schema bound by a committed CREATE EXTERNAL SCHEMA.+ if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null) {+ if (type == 'E') {+ spectrumInterceptor.preparation().finishCycle(relationalBackend);+ }+ if (type == 'C' && "ROLLBACK\0".equals(new String(body, StandardCharsets.UTF_8))) {+ // Loads are dropped on every rollback; bindings and Glue tables only once ReadyForQuery+ // reports the transaction closed, because ROLLBACK TO SAVEPOINT keeps it open.+ rolledBackInCycle = true;+ spectrumInterceptor.preparation().discardPendingLoads(relationalBackend);+ }+ if (type == 'Z' && body.length > 0 && body[0] == 'I') {+ if (rolledBackInCycle) {+ spectrumInterceptor.preparation().finishCycle(relationalBackend);+ } else {+ spectrumInterceptor.preparation().finishCycle(relationalBackend, true);+ }+ }+ if (type == 'Z') {+ rolledBackInCycle = false;+ }+ }+ if (type == 'E') {+ relationalBackend.onBackendError();+ }+ coordinator.onBackendFrame(type, body);+ if (type == 'Z') {+ relationalBackend.onSync();+ }+ });try {InputStream backendIn = backend.getInputStream();OutputStream clientOut = client.getOutputStream();@@ -770,6 +878,7 @@ public class RedshiftInterceptingBridge {private long declaredBodyLength = 0;private char pendingType = 0;private byte readyStatus;+ private byte[] commandTag = EMPTY_BODY;WireFrameTracker(FrameCompletion onFrame) {this.onFrame = onFrame;@@ -786,6 +895,9 @@ public class RedshiftInterceptingBridge {if (pendingType == 'Z' && declaredBodyLength == 1) {readyStatus = (byte) current;}+ if (pendingType == 'C' && commandTag.length > 0) {+ commandTag[(int) (declaredBodyLength - bodyRemaining)] = (byte) current;+ }bodyRemaining--;if (bodyRemaining == 0) {publishCompletion();@@ -799,6 +911,7 @@ public class RedshiftInterceptingBridge {pendingType = (char) (header[0] & 0xFF);bodyRemaining = Math.max(0, msgLen - 4);declaredBodyLength = bodyRemaining;+ commandTag = pendingType == 'C' && bodyRemaining <= 128 ? new byte[(int) bodyRemaining] : EMPTY_BODY;headerBytesSeen = 0;if (bodyRemaining == 0) {publishCompletion();@@ -809,7 +922,7 @@ public class RedshiftInterceptingBridge {private void publishCompletion() {byte[] body = pendingType == 'Z' && declaredBodyLength == 1- ? new byte[]{readyStatus} : new byte[0];+ ? new byte[]{readyStatus} : pendingType == 'C' ? commandTag : EMPTY_BODY;onFrame.accept(pendingType, body);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.javaindex 0fafb683d5..cd09cb0a38 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java@@ -5,8 +5,8 @@ import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;-import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumInterceptor;+import io.github.hectorvent.floci.services.s3.S3Service;import io.quarkus.runtime.ShutdownEvent;import jakarta.enterprise.context.ApplicationScoped;import jakarta.enterprise.event.Observes;@@ -76,6 +76,7 @@ public class RedshiftProxyManager {// Make sure the self-signed proxy certificate covers the host clients will connect to,// so sslmode=prefer/require handshakes succeed.tlsCertificates.ensureHost(advertisedHost);+ invalidateSpectrumRuntime(relayKey);EmulatorConfig.RedshiftServiceConfig redshiftConfig = config.services().redshift();String clusterAccountId = relayKey.substring(0, relayKey.indexOf(':'));RedshiftAuthProxy proxy = new RedshiftAuthProxy(@@ -138,6 +139,20 @@ public class RedshiftProxyManager {proxies.remove(relayKey);LOG.infov("Stopped Redshift proxy for cluster {0}", relayKey);}+ invalidateSpectrumRuntime(relayKey);+ }++ public void forgetSpectrumRuntime(String relayKey) {+ if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null) {+ String accountId = relayKey.substring(0, relayKey.indexOf(':'));+ spectrumInterceptor.preparation().forgetRuntime(accountId, relayKey);+ }+ }++ private void invalidateSpectrumRuntime(String relayKey) {+ if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null) {+ spectrumInterceptor.preparation().invalidateRuntime(relayKey);+ }}public synchronized void stopAll() {diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.javaindex 11e541fc26..91b5f79c42 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java@@ -3,6 +3,10 @@ package io.github.hectorvent.floci.services.redshift.spectrum;import java.io.InputStream;public interface BackendSql {+ default boolean permitsCachePublication() {+ return true;+ }+void execute(String sql);long copyIn(String copySql, InputStream data);}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.javaindex 66d6c10f63..39eddc4569 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java@@ -68,6 +68,32 @@ public class ExternalTableMaterializer {private final ConcurrentHashMap<String, String> schemaSignatures = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, ReentrantLock> locks = new ConcurrentHashMap<>();private final Set<String> sweptAccounts = ConcurrentHashMap.newKeySet();+ private final Map<BackendSql, Map<String, PendingLoad>> pendingLoads = new ConcurrentHashMap<>();+ private final Map<String, PendingLoad> latestLoads = new ConcurrentHashMap<>();++ private record PendingLoad(String fingerprint, String definitions) { }++ public void finishCycle(BackendSql backend) {+ pendingLoads.remove(backend);+ }++ public void finishCycle(BackendSql backend, boolean committed) {+ Map<String, PendingLoad> completed = pendingLoads.remove(backend);+ if (committed && completed != null) {+ completed.forEach((key, loaded) -> {+ ReentrantLock lock = locks.computeIfAbsent(key, ignored -> new ReentrantLock());+ lock.lock();+ try {+ if (latestLoads.get(key) == loaded) {+ fingerprints.put(key, loaded.fingerprint());+ schemaSignatures.put(key, loaded.definitions());+ }+ } finally {+ lock.unlock();+ }+ });+ }+ }public ExternalTableMaterializer(FlociDuckClient duckClient, GlueService glueService, S3Service s3Service,IamService iamService, EmulatorConfig config) {@@ -107,6 +133,12 @@ public class ExternalTableMaterializer {// An Iceberg snapshot can change through its metadata or manifests while the Glue table and the// listed objects stay the same, so no fingerprint can vouch for it: it is read on every query.boolean cacheable = !session.inTransaction() && !GlueTableResolver.isIcebergTable(table);+ Map<String, PendingLoad> localLoads = backend.permitsCachePublication() ? Map.of()+ : pendingLoads.computeIfAbsent(backend, ignored -> new ConcurrentHashMap<>());+ PendingLoad local = localLoads.get(cacheKey);+ if (!GlueTableResolver.isIcebergTable(table) && local != null && fingerprint.equals(local.fingerprint())) {+ return Outcome.CURRENT;+ }if (cacheable && fingerprint.equals(fingerprints.get(cacheKey))) {return Outcome.CURRENT;}@@ -116,9 +148,18 @@ public class ExternalTableMaterializer {if (cacheable && fingerprint.equals(fingerprints.get(cacheKey))) {return Outcome.CURRENT;}- String previousSignature = schemaSignatures.get(cacheKey);+ String previousSignature = local == null ? schemaSignatures.get(cacheKey) : local.definitions();+ latestLoads.remove(cacheKey);String definitions = load(backend, session, binding, table, sources, previousSignature);- if (session.inTransaction()) {+ if (!backend.permitsCachePublication()) {+ fingerprints.remove(cacheKey);+ PendingLoad loaded = new PendingLoad(fingerprint, definitions);+ localLoads.put(cacheKey, loaded);+ latestLoads.put(cacheKey, loaded);+ if (!definitions.equals(schemaSignatures.get(cacheKey))) {+ schemaSignatures.remove(cacheKey);+ }+ } else if (session.inTransaction()) {// Transactional DDL may roll back, so the rows cannot be trusted afterwards. The schema// signature stays when this load kept the committed columns, so a view that depends on the// table can still be refilled in place; a replaced table may roll back to the old columns.@@ -182,6 +223,7 @@ public class ExternalTableMaterializer {private void forgetMatching(Predicate<String> matches) {fingerprints.keySet().removeIf(matches);schemaSignatures.keySet().removeIf(matches);+ latestLoads.keySet().removeIf(matches);// Keep locks: removing a held lock lets another thread create a second lock for the same table.}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresExtendedBackendSession.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresExtendedBackendSession.javanew file mode 100644index 0000000000..365ce179cb--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresExtendedBackendSession.java@@ -0,0 +1,203 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import java.io.ByteArrayOutputStream;+import java.io.DataInputStream;+import java.io.DataOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.net.Socket;+import java.nio.charset.StandardCharsets;+import java.util.UUID;++/** Runs preparation without ending the client's transaction or destroying its portals. */+public final class PostgresExtendedBackendSession implements BackendSql {+ private final Socket backend;+ private String name;+ private volatile boolean failedUntilSync;++ public PostgresExtendedBackendSession(Socket backend) {+ this.backend = backend;+ }++ @Override+ public boolean permitsCachePublication() {+ return false;+ }++ /** The client's own pipelined statement failed; the backend discards everything until Sync. */+ public void onBackendError() {+ failedUntilSync = true;+ }++ public void onSync() {+ failedUntilSync = false;+ }++ @Override+ public void execute(String sql) {+ try {+ start(atomicCommand(sql));+ closeAndFlush();+ awaitClosed();+ } catch (IOException exception) {+ throw failure(exception);+ }+ }++ public static String atomicCommand(String sql) {+ if (!sql.startsWith("CREATE ") && !sql.startsWith("DROP ")) {+ return sql;+ }+ String delimiter = "$floci_" + UUID.randomUUID().toString().replace("-", "") + "$";+ return "DO " + delimiter + " BEGIN " + sql + "; END " + delimiter;+ }++ @Override+ public long copyIn(String sql, InputStream data) {+ try {+ start(sql);+ frame('H', new byte[0]);+ backend.getOutputStream().flush();+ readUntil('G');+ byte[] chunk = new byte[8192];+ try {+ int count;+ while ((count = data.read(chunk)) >= 0) {+ if (count > 0) {+ frame('d', chunk, count);+ }+ }+ } catch (IOException exception) {+ frame('f', "Unable to read external data\0".getBytes(StandardCharsets.UTF_8));+ backend.getOutputStream().flush();+ try {+ readUntil('C');+ } catch (SpectrumReadException expected) {+ // COPY failure leaves recovery to the client's Sync.+ exception.addSuppressed(expected);+ }+ throw exception;+ }+ frame('c', new byte[0]);+ closeAndFlush();+ String tag = awaitClosed();+ return Long.parseLong(tag.substring(tag.lastIndexOf(' ') + 1).trim());+ } catch (IOException exception) {+ throw failure(exception);+ }+ }++ private void start(String sql) throws IOException {+ if (failedUntilSync) {+ throw new SpectrumReadException("25P02", "Preparation is waiting for Sync after a backend error");+ }+ name = "floci_prepare_" + UUID.randomUUID().toString().replace("-", "");+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();+ DataOutputStream body = new DataOutputStream(bytes);+ cstring(body, name);+ cstring(body, sql);+ body.writeShort(0);+ frame('P', bytes.toByteArray());+ bytes.reset();+ cstring(body, name);+ cstring(body, name);+ body.writeShort(0);+ body.writeShort(0);+ body.writeShort(0);+ frame('B', bytes.toByteArray());+ bytes.reset();+ cstring(body, name);+ body.writeInt(0);+ frame('E', bytes.toByteArray());+ }++ private void closeAndFlush() throws IOException {+ frame('C', ("P" + name + "\0").getBytes(StandardCharsets.UTF_8));+ frame('C', ("S" + name + "\0").getBytes(StandardCharsets.UTF_8));+ frame('H', new byte[0]);+ backend.getOutputStream().flush();+ }++ private String awaitClosed() throws IOException {+ String tag = "";+ for (int closed = 0; closed < 2;) {+ Response response = read();+ if (response.type() == '3') {+ closed++;+ } else if (response.type() == 'C') {+ tag = new String(response.body(), StandardCharsets.UTF_8).replace("\0", "");+ }+ }+ return tag;+ }++ private void readUntil(char type) throws IOException {+ Response response;+ do {+ // ParseComplete, BindComplete and asynchronous messages precede the response.+ response = read();+ if (response.type() == 'Z') {+ throw new IOException("Backend ended preparation before sending " + type);+ }+ } while (response.type() != type);+ }++ private Response read() throws IOException {+ DataInputStream input = new DataInputStream(backend.getInputStream());+ char type = (char) input.readUnsignedByte();+ int length = input.readInt();+ if (length < 4 || length > 64 * 1024 * 1024) {+ throw new IOException("Invalid preparation response length");+ }+ byte[] body = input.readNBytes(length - 4);+ if (body.length != length - 4) {+ throw new IOException("Backend closed during preparation");+ }+ if (type == 'E') {+ failedUntilSync = true;+ String state = "58030";+ String message = "External query preparation failed";+ for (int offset = 0; offset < body.length && body[offset] != 0;) {+ char field = (char) body[offset++];+ int start = offset;+ while (offset < body.length && body[offset] != 0) {+ offset++;+ }+ String value = new String(body, start, offset - start, StandardCharsets.UTF_8);+ if (field == 'C') {+ state = value;+ } else if (field == 'M') {+ message = value;+ }+ offset++;+ }+ throw new SpectrumReadException(state, message);+ }+ return new Response(type, body);+ }++ private void frame(char type, byte[] body) throws IOException {+ frame(type, body, body.length);+ }++ /** Writes the frame in one call; a split write would put a header packet on the wire of its own. */+ private void frame(char type, byte[] body, int length) throws IOException {+ ByteArrayOutputStream bytes = new ByteArrayOutputStream(length + 5);+ DataOutputStream output = new DataOutputStream(bytes);+ output.writeByte(type);+ output.writeInt(length + 4);+ output.write(body, 0, length);+ backend.getOutputStream().write(bytes.toByteArray());+ }++ private static void cstring(DataOutputStream output, String text) throws IOException {+ output.write(text.getBytes(StandardCharsets.UTF_8));+ output.writeByte(0);+ }++ private static SpectrumReadException failure(IOException exception) {+ return new SpectrumReadException("58030", "Unable to prepare external query", exception);+ }++ private record Response(char type, byte[] body) { }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.javaindex 13820d901b..afa68e92ed 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.redshift.spectrum;import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;import java.net.Socket;import java.util.Optional;@@ -14,16 +15,48 @@ public final class SpectrumInterceptor {private final SpectrumQueryRewriter queryRewriter;private final SpectrumS3Reader reader;private final SpectrumMaterializer materializer;+ private final SpectrumQueryPreparation preparation;public SpectrumInterceptor(SpectrumCatalog catalog, SpectrumStatementParser statementParser,SpectrumQueryClassifier queryClassifier, SpectrumQueryRewriter queryRewriter,SpectrumS3Reader reader, SpectrumMaterializer materializer) {+ this(catalog, statementParser, queryClassifier, queryRewriter, reader, materializer, null);+ }++ @Inject+ public SpectrumInterceptor(SpectrumCatalog catalog, SpectrumStatementParser statementParser,+ SpectrumQueryClassifier queryClassifier, SpectrumQueryRewriter queryRewriter,+ SpectrumS3Reader reader, SpectrumMaterializer materializer,+ SpectrumQueryPreparation preparation) {this.catalog = catalog;this.statementParser = statementParser;this.queryClassifier = queryClassifier;this.queryRewriter = queryRewriter;this.reader = reader;this.materializer = materializer;+ this.preparation = preparation;+ }++ public SpectrumQueryPreparation preparation() {+ return preparation;+ }++ public Plan plan(String sql, SpectrumSession session) {+ if (preparation != null && preparation.handlesDdl(sql, session)) {+ return new Plan.ExternalDdl(sql, session);+ }+ if (preparation != null && preparation.referencesExternal(sql, session)) {+ return new Plan.Forward();+ }+ return plan(sql, session.accountId(), session.databaseName());+ }++ public Decision intercept(String sql, SpectrumSession session, Socket backend) {+ Plan plan = plan(sql, session);+ if (plan instanceof Plan.Forward && preparation != null) {+ preparation.prepare(sql, session, new PostgresBackendSession(backend));+ }+ return execute(plan, session.accountId(), session.databaseName(), backend);}public Decision intercept(String sql, String accountId, String databaseName, Socket backend) {@@ -54,6 +87,12 @@ public final class SpectrumInterceptor {public Decision execute(Plan plan, String accountId, String databaseName, Socket backend) {switch (plan) {+ case Plan.ExternalDdl(String sql, SpectrumSession session) -> {+ if (preparation.prepare(sql, session, new PostgresBackendSession(backend))) {+ return new Decision.Handled();+ }+ return execute(plan(sql, accountId, databaseName), accountId, databaseName, backend);+ }case Plan.Ddl(SpectrumStatement parsed) -> {switch (parsed) {case SpectrumStatement.CreateSchema schema -> catalog.createSchema(new SpectrumExternalSchema(@@ -91,7 +130,9 @@ public final class SpectrumInterceptor {materializer.cleanup(backend, materialization);}- public sealed interface Plan permits Plan.Ddl, Plan.Forward, Plan.Query {+ public sealed interface Plan permits Plan.Ddl, Plan.Forward, Plan.Query, Plan.ExternalDdl {+ record ExternalDdl(String sql, SpectrumSession session) implements Plan {+ }record Ddl(SpectrumStatement statement) implements Plan {}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparation.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparation.javanew file mode 100644index 0000000000..8a4c3a0d3b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparation.java@@ -0,0 +1,195 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.glue.GlueService;+import io.github.hectorvent.floci.services.glue.model.Database;+import jakarta.enterprise.context.ApplicationScoped;+import org.jboss.logging.Logger;++import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.CopyOnWriteArrayList;++@ApplicationScoped+public class SpectrumQueryPreparation {+ private static final Logger LOG = Logger.getLogger(SpectrumQueryPreparation.class);+ private final ExternalCatalogRegistry registry;+ private final ExternalTableMaterializer materializer;+ private final ExternalStatementParser parser;+ private final GlueService glue;+ private final ExternalMetadataWriter metadata;+ private final Map<BackendSql, List<ExternalSchemaBinding>> pendingBindings = new ConcurrentHashMap<>();++ public SpectrumQueryPreparation(ExternalCatalogRegistry registry, ExternalTableMaterializer materializer,+ ExternalStatementParser parser, GlueService glue, ExternalMetadataWriter metadata) {+ this.registry = registry;+ this.materializer = materializer;+ this.parser = parser;+ this.glue = glue;+ this.metadata = metadata;+ }++ public boolean prepare(String sql, SpectrumSession session, BackendSql backend) {+ try {+ return RequestScopes.callAs(session.accountId(), () -> prepareInScope(sql, session, backend));+ } catch (AwsException exception) {+ String state = switch (exception.getErrorCode()) {+ case "AlreadyExistsException" -> "42P07";+ case "EntityNotFoundException" -> "42P01";+ case "AccessDenied", "AccessDeniedException" -> "42501";+ case "InvalidInputException" -> "42601";+ default -> "58030";+ };+ throw new SpectrumSqlException(state, exception.getMessage());+ }+ }++ public boolean referencesExternal(String sql, SpectrumSession session) {+ Set<String> schemas = bindings(session).keySet();+ return !ExternalReferenceScanner.scan(sql, schemas).isEmpty()+ || ExternalReferenceScanner.writeTarget(sql, schemas).isPresent();+ }++ public boolean handlesDdl(String sql, SpectrumSession session) {+ Optional<ExternalStatement> statement = parser.parse(sql);+ return statement.orElse(null) instanceof ExternalStatement.CreateSchema+ || statement.orElse(null) instanceof ExternalStatement.CreateTable table+ && (session.inTransaction() || bindings(session).containsKey(table.schemaName()));+ }++ private boolean prepareInScope(String sql, SpectrumSession session, BackendSql backend) {+ Optional<ExternalStatement> statement = parser.parse(sql);+ if (statement.orElse(null) instanceof ExternalStatement.CreateSchema schema) {+ return createSchema(schema, session, backend);+ }+ Map<String, ExternalSchemaBinding> bindings = bindings(session);+ if (statement.orElse(null) instanceof ExternalStatement.CreateTable && session.inTransaction()) {+ // Legacy-catalog tables are saved immediately too, so a rollback would leave them behind.+ throw new SpectrumSqlException("25001", "CREATE EXTERNAL TABLE cannot run inside a transaction block");+ }+ if (statement.orElse(null) instanceof ExternalStatement.CreateTable table && bindings.containsKey(table.schemaName())) {+ ExternalSchemaBinding binding = bindings.get(table.schemaName());+ glue.createTable(binding.glueDatabase(), GlueTableBuilder.toGlueTable(table));+ metadata.refresh(backend, session.accountId(), binding);+ return true;+ }+ Optional<ExternalReferenceScanner.Reference> write = ExternalReferenceScanner.writeTarget(sql, bindings.keySet());+ if (write.isPresent()) {+ throw new SpectrumSqlException("0A000", "Writes to external relation \""+ + write.get().schema() + "." + write.get().table() + "\" are not supported");+ }+ Set<String> refreshed = new HashSet<>();+ for (ExternalReferenceScanner.Reference reference : ExternalReferenceScanner.scan(sql, bindings.keySet())) {+ ExternalSchemaBinding binding = bindings.get(reference.schema());+ ExternalTableMaterializer.Outcome outcome = materializer.ensureCurrent(backend, session, binding, reference.table());+ if (outcome == ExternalTableMaterializer.Outcome.NOT_EXTERNAL) {+ throw new SpectrumSqlException("42P01", "external relation \"" + reference.schema()+ + "." + reference.table() + "\" does not exist");+ }+ if (outcome == ExternalTableMaterializer.Outcome.LOADED && refreshed.add(binding.schemaName())) {+ metadata.refresh(backend, session.accountId(), binding);+ }+ }+ return false;+ }++ private boolean createSchema(ExternalStatement.CreateSchema schema, SpectrumSession session, BackendSql backend) {+ if (session.inTransaction()) {+ throw new SpectrumSqlException("0A000", "CREATE EXTERNAL SCHEMA inside a transaction is not supported");+ }+ Optional<ExternalSchemaBinding> existing = registry.find(session.accountId(), session.clusterKey(),+ session.databaseName(), schema.schemaName());+ if (existing.isPresent()) {+ if (schema.ifNotExists()) {+ return true;+ }+ throw new SpectrumSqlException("42P06", "schema \"" + schema.schemaName() + "\" already exists");+ }+ try {+ glue.getDatabase(schema.glueDatabase());+ } catch (AwsException exception) {+ if (!"EntityNotFoundException".equals(exception.getErrorCode())) {+ throw exception;+ }+ if (!schema.createDatabaseIfNotExists()) {+ // Preserve the legacy catalog path for schemas without a Glue database.+ return false;+ }+ Database database = new Database();+ database.setName(schema.glueDatabase());+ glue.createDatabase(database);+ }+ ExternalSchemaBinding binding = new ExternalSchemaBinding(session.accountId(), session.clusterKey(),+ session.databaseName(), schema.schemaName(), schema.glueDatabase(), schema.iamRoleArn());+ backend.execute("CREATE SCHEMA " + ExternalTableMaterializer.quote(schema.schemaName()));+ try {+ metadata.refresh(backend, session.accountId(), binding);+ bindWhenCommitted(backend, binding);+ } catch (RuntimeException exception) {+ try {+ backend.execute("DROP SCHEMA " + ExternalTableMaterializer.quote(schema.schemaName()) + " CASCADE");+ } catch (RuntimeException cleanupFailure) {+ LOG.warnv(cleanupFailure, "Could not clean up failed external schema {0}", schema.schemaName());+ }+ throw exception;+ }+ return true;+ }++ private Map<String, ExternalSchemaBinding> bindings(SpectrumSession session) {+ Map<String, ExternalSchemaBinding> bindings = new LinkedHashMap<>();+ for (ExternalSchemaBinding binding : registry.list(session.accountId(), session.clusterKey(), session.databaseName())) {+ bindings.put(binding.schemaName(), binding);+ }+ return bindings;+ }++ public void forgetRuntime(String accountId, String clusterKey) {+ registry.removeCluster(accountId, clusterKey);+ materializer.forgetCluster(clusterKey);+ }++ public void invalidateRuntime(String clusterKey) {+ materializer.forgetCluster(clusterKey);+ }++ /**+ * A backend that shares the client's transaction can still roll the schema back, so the binding is held+ * until {@link #finishCycle(BackendSql, boolean)} sees the commit. Any other backend has committed already.+ */+ private void bindWhenCommitted(BackendSql backend, ExternalSchemaBinding binding) {+ if (backend.permitsCachePublication()) {+ registry.bind(binding);+ return;+ }+ pendingBindings.computeIfAbsent(backend, ignored -> new CopyOnWriteArrayList<>()).add(binding);+ }++ /** The transaction rolled back: drop everything staged for it. */+ public void finishCycle(BackendSql backend) {+ pendingBindings.remove(backend);+ materializer.finishCycle(backend);+ }++ public void finishCycle(BackendSql backend, boolean committed) {+ List<ExternalSchemaBinding> completed = pendingBindings.remove(backend);+ if (committed && completed != null) {+ completed.forEach(registry::bind);+ }+ materializer.finishCycle(backend, committed);+ }++ /**+ * A rollback to a savepoint leaves the transaction open, so only the loads are dropped: they may have been+ * made after the savepoint, and their fingerprints would no longer match the backend tables.+ */+ public void discardPendingLoads(BackendSql backend) {+ materializer.finishCycle(backend);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSql.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSql.javanew file mode 100644index 0000000000..5b697b3bf4--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSql.java@@ -0,0 +1,44 @@+package io.github.hectorvent.floci.services.redshiftdata;++import io.github.hectorvent.floci.services.redshift.spectrum.BackendSql;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumReadException;+import org.postgresql.PGConnection;++import java.io.IOException;+import java.io.InputStream;+import java.sql.Connection;+import java.sql.SQLException;+import java.sql.Statement;++final class RedshiftDataBackendSql implements BackendSql {+ private final Connection connection;++ RedshiftDataBackendSql(Connection connection) {+ this.connection = connection;+ }++ @Override+ public void execute(String sql) {+ try (Statement statement = connection.createStatement()) {+ statement.execute(sql);+ } catch (SQLException exception) {+ throw failure(exception);+ }+ }++ @Override+ public long copyIn(String copySql, InputStream data) {+ try {+ return connection.unwrap(PGConnection.class).getCopyAPI().copyIn(copySql, data);+ } catch (SQLException exception) {+ throw failure(exception);+ } catch (IOException exception) {+ throw new SpectrumReadException("58030", "Unable to stream external table data", exception);+ }+ }++ private static SpectrumReadException failure(SQLException exception) {+ String state = exception.getSQLState() == null ? "58030" : exception.getSQLState();+ return new SpectrumReadException(state, exception.getMessage(), exception);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.javaindex e2029564d5..0ef79e221b 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java@@ -8,6 +8,8 @@ import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.services.redshift.RedshiftCredentialBroker;import io.github.hectorvent.floci.services.redshift.RedshiftService;import io.github.hectorvent.floci.services.redshift.model.Cluster;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSession;+import io.github.hectorvent.floci.services.redshiftserverless.RedshiftServerlessRuntime;import io.github.hectorvent.floci.services.redshiftserverless.RedshiftServerlessService;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;@@ -65,17 +67,21 @@ class RedshiftDataResourceResolver {}RedshiftServerlessService.WorkgroupTarget workgroup =serverlessService.getWorkgroupTarget(request.get("WorkgroupName").asText(), region);+ String accountId = AwsArnUtils.parse(workgroup.arn()).accountId();+ SpectrumSession spectrum = new SpectrumSession(accountId,+ accountId + ":" + RedshiftServerlessRuntime.backendId(region, workgroup.workgroupName()), database,+ workgroup.iamRoleArns(), false);if (!database.equals(workgroup.database())) {throw validation("Database " + database + " does not exist in workgroup " + workgroup.workgroupName()+ "; the only database is " + workgroup.database() + ".");}if (!hasText(request, "SecretArn")) {return new DatabaseTarget(workgroup.arn(), workgroup.host(), workgroup.port(), database,- workgroup.masterUsername(), workgroup.masterPassword());+ workgroup.masterUsername(), workgroup.masterPassword(), spectrum);}Credentials creds = secretCredentials(request.get("SecretArn").asText(), region);return new DatabaseTarget(workgroup.arn(), workgroup.host(), workgroup.port(), database,- creds.username(), creds.password());+ creds.username(), creds.password(), spectrum);}private Credentials secretCredentials(String secretArn, String region) {@@ -146,7 +152,10 @@ class RedshiftDataResourceResolver {if (host == null || host.isBlank() || port <= 0) {throw validation("Cluster runtime is not available for Data API execution.");}- return new DatabaseTarget(arn, host, port, database, user, password);+ String accountId = AwsArnUtils.parse(arn).accountId();+ SpectrumSession spectrum = new SpectrumSession(accountId, accountId + ":" + cluster.getClusterIdentifier(),+ database, cluster.getIamRoleArns(), false);+ return new DatabaseTarget(arn, host, port, database, user, password, spectrum);}private Credentials parseCredentials(String secretString) {@@ -196,7 +205,11 @@ class RedshiftDataResourceResolver {return value == null || value.isNull() ? null : value.asText();}- record DatabaseTarget(String arn, String host, int port, String database, String user, String password) {+ record DatabaseTarget(String arn, String host, int port, String database, String user, String password,+ SpectrumSession spectrum) {+ DatabaseTarget(String arn, String host, int port, String database, String user, String password) {+ this(arn, host, port, database, user, password, null);+ }}private record Credentials(String username, String password) {diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.javaindex 5c0382d39a..2aa84383d0 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java@@ -7,6 +7,11 @@ import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.common.SqlParameterParser.ParsedSql;+import io.github.hectorvent.floci.services.redshift.spectrum.BackendSql;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumQueryPreparation;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumReadException;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSession;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSqlException;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -33,16 +38,25 @@ public class RedshiftDataService implements Resettable {private final RedshiftDataConnectionFactory connectionFactory;private final RedshiftDataStatementStore store;private final ObjectMapper objectMapper;+ private final SpectrumQueryPreparation preparation;- @Injectpublic RedshiftDataService(RedshiftDataResourceResolver resolver,RedshiftDataConnectionFactory connectionFactory,RedshiftDataStatementStore store,ObjectMapper objectMapper) {+ this(resolver, connectionFactory, store, objectMapper, null);+ }++ @Inject+ public RedshiftDataService(RedshiftDataResourceResolver resolver,+ RedshiftDataConnectionFactory connectionFactory,+ RedshiftDataStatementStore store, ObjectMapper objectMapper,+ SpectrumQueryPreparation preparation) {this.resolver = resolver;this.connectionFactory = connectionFactory;this.store = store;this.objectMapper = objectMapper;+ this.preparation = preparation;}@Override@@ -77,7 +91,7 @@ public class RedshiftDataService implements Resettable {RedshiftDataResourceResolver.DatabaseTarget target,String sql, Map<String, String> parameters) {try (Connection connection = connectionFactory.open(target)) {- runOnConnection(stored, connection, sql, parameters);+ runOnConnection(stored, connection, sql, parameters, target.spectrum());} catch (SQLException e) {stored.status = RedshiftDataStatementStore.Status.FAILED;stored.error = e.getMessage();@@ -104,8 +118,25 @@ public class RedshiftDataService implements Resettable {}private void runOnConnection(RedshiftDataStatementStore.StoredStatement stored,- Connection connection, String sql, Map<String, String> parameters)+ Connection connection, String sql, Map<String, String> parameters, SpectrumSession context)throws SQLException {+ if (preparation != null && context != null) {+ SpectrumSession session = new SpectrumSession(context.accountId(), context.clusterKey(), context.databaseName(),+ context.iamRoleArns(), !connection.getAutoCommit());+ try {+ if (preparation.prepare(sql, session, new RedshiftDataBackendSql(connection))) {+ stored.status = RedshiftDataStatementStore.Status.FINISHED;+ return;+ }+ } catch (SpectrumSqlException exception) {+ throw new SQLException(exception.getMessage(), exception.sqlState(), exception);+ } catch (SpectrumReadException exception) {+ throw new SQLException(exception.getMessage(), exception.sqlState(), exception);+ } catch (RuntimeException exception) {+ // Anything else from Glue or the read path must fail the statement, not leave it running.+ throw new SQLException("External query preparation failed: " + exception.getMessage(), "58030", exception);+ }+ }ParsedSql parsed = RedshiftDataSqlParameters.parse(sql);long t0 = System.nanoTime();try (PreparedStatement statement = connection.prepareStatement(parsed.sql())) {@@ -309,44 +340,57 @@ public class RedshiftDataService implements Resettable {long totalDuration = 0;try (Connection connection = connectionFactory.open(target)) {- connection.setAutoCommit(false);- boolean failed = false;- for (int n = 0; n < sqls.size(); n++) {- RedshiftDataStatementStore.StoredStatement sub = new RedshiftDataStatementStore.StoredStatement();- sub.id = parent.id + ":" + (n + 1);- sub.sql = sqls.get(n);- sub.sqls = List.of(sqls.get(n));- sub.clusterIdentifier = parent.clusterIdentifier;- sub.workgroupName = parent.workgroupName;- sub.database = parent.database;- sub.dbUser = parent.dbUser;- sub.resultFormat = parent.resultFormat;- Instant now = Instant.now();- sub.createdAt = now;- sub.updatedAt = now;- try {- runOnConnection(sub, connection, sqls.get(n), Map.of());- } catch (SQLException e) {- sub.status = RedshiftDataStatementStore.Status.FAILED;- sub.error = e.getMessage();+ BackendSql batchBackend = new RedshiftDataBackendSql(connection);+ boolean committed = false;+ try {+ connection.setAutoCommit(false);+ boolean failed = false;+ for (int n = 0; n < sqls.size(); n++) {+ RedshiftDataStatementStore.StoredStatement sub = new RedshiftDataStatementStore.StoredStatement();+ sub.id = parent.id + ":" + (n + 1);+ sub.sql = sqls.get(n);+ sub.sqls = List.of(sqls.get(n));+ sub.clusterIdentifier = parent.clusterIdentifier;+ sub.workgroupName = parent.workgroupName;+ sub.database = parent.database;+ sub.dbUser = parent.dbUser;+ sub.resultFormat = parent.resultFormat;+ Instant now = Instant.now();+ sub.createdAt = now;+ sub.updatedAt = now;+ try {+ runOnConnection(sub, connection, sqls.get(n), Map.of(), target.spectrum());+ } catch (SQLException e) {+ sub.status = RedshiftDataStatementStore.Status.FAILED;+ sub.error = e.getMessage();+ sub.updatedAt = Instant.now();+ parent.subStatements.add(sub);+ store.put(sub);+ totalDuration += sub.durationNanos;+ parent.status = RedshiftDataStatementStore.Status.FAILED;+ parent.error = e.getMessage();+ connection.rollback();+ failed = true;+ break;+ }sub.updatedAt = Instant.now();parent.subStatements.add(sub);store.put(sub);totalDuration += sub.durationNanos;- parent.status = RedshiftDataStatementStore.Status.FAILED;- parent.error = e.getMessage();- connection.rollback();- failed = true;- break;}- sub.updatedAt = Instant.now();- parent.subStatements.add(sub);- store.put(sub);- totalDuration += sub.durationNanos;- }- if (!failed) {- connection.commit();- parent.status = RedshiftDataStatementStore.Status.FINISHED;+ if (!failed) {+ connection.commit();+ committed = true;+ parent.status = RedshiftDataStatementStore.Status.FINISHED;+ }+ } finally {+ if (preparation != null) {+ if (committed) {+ preparation.finishCycle(batchBackend, true);+ } else {+ preparation.finishCycle(batchBackend);+ }+ }}} catch (SQLException e) {parent.status = RedshiftDataStatementStore.Status.FAILED;diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.javaindex b1c7076ad8..7465f6c996 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java@@ -89,6 +89,7 @@ public class RedshiftServerlessRuntime {String backendId = backendId(region, workgroupName);proxyManager.stopProxy(relayKey(accountId, backendId));containerManager.stop(accountId, backendId);+ proxyManager.forgetSpectrumRuntime(relayKey(accountId, backendId));credentialBroker.revokeCluster(accountId, backendId);}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessService.java b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessService.javaindex 883ace9243..7e8737ccf9 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessService.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessService.java@@ -562,12 +562,20 @@ public class RedshiftServerlessService implements Resettable {}return new WorkgroupTarget(workgroup.getWorkgroupArn(), workgroup.getWorkgroupName(),workgroup.getRuntimeHost(), workgroup.getRuntimePort(), namespace.getDbName(),- workgroup.getMasterUsername(), namespace.getAdminUserPassword(), adminUserOf(namespace));+ workgroup.getMasterUsername(), namespace.getAdminUserPassword(), adminUserOf(namespace), namespace.getIamRoles());}/** Connection details of a workgroup's backend; {@code adminUsername} is the name clients log in with. */public record WorkgroupTarget(String arn, String workgroupName, String host, int port, String database,- String masterUsername, String masterPassword, String adminUsername) {+ String masterUsername, String masterPassword, String adminUsername, List<String> iamRoleArns) {+ public WorkgroupTarget(String arn, String workgroupName, String host, int port, String database,+ String masterUsername, String masterPassword, String adminUsername) {+ this(arn, workgroupName, host, port, database, masterUsername, masterPassword, adminUsername, List.of());+ }++ public WorkgroupTarget {+ iamRoleArns = iamRoleArns == null ? List.of() : List.copyOf(iamRoleArns);+ }}private Workgroup resolveWorkgroup(String nameOrArn, String region) {diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 1ab7b0c624..91f5ae168f 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -65,6 +65,7 @@ quarkus:- eks/*.json- aws/*.json- cloudformation/*.json+ - redshift/*.sql- i18n/*.properties- org/apache/velocity/runtime/defaults/*.properties# DriverManager finds the JDBC drivers through ServiceLoader, and the MySQLdiff --git a/src/main/resources/redshift/bootstrap-catalog.sql b/src/main/resources/redshift/bootstrap-catalog.sqlindex fbeaab20d7..f85fd408bd 100644--- a/src/main/resources/redshift/bootstrap-catalog.sql+++ b/src/main/resources/redshift/bootstrap-catalog.sql@@ -359,6 +359,62 @@ WHERE c.relkind IN ('r', 'm')AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast');-- Grant access to all database users (including IAM temporary users)+CREATE SCHEMA IF NOT EXISTS floci_internal;+CREATE TABLE IF NOT EXISTS floci_internal.external_catalog (+ schema_name text PRIMARY KEY,+ payload jsonb NOT NULL+);++CREATE OR REPLACE FUNCTION floci_internal.refresh_external_catalog(schema_name text, payload jsonb)+RETURNS void LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$+ INSERT INTO floci_internal.external_catalog VALUES ($1, $2)+ ON CONFLICT (schema_name) DO UPDATE SET payload = EXCLUDED.payload;+$$;++CREATE OR REPLACE FUNCTION floci_internal.purge_external_schema(schema_name text)+RETURNS void LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$+ DELETE FROM floci_internal.external_catalog WHERE schema_name = $1;+$$;++CREATE OR REPLACE VIEW pg_catalog.svv_external_schemas AS+SELECT n.oid AS esoid, 1::smallint AS eskind, n.nspname AS schemaname,+ n.nspowner::integer AS esowner, c.payload->>'databasename' AS databasename,+ c.payload->>'esoptions' AS esoptions+FROM floci_internal.external_catalog c+JOIN pg_catalog.pg_namespace n ON n.nspname = c.schema_name+WHERE has_schema_privilege(n.oid, 'USAGE');++CREATE OR REPLACE VIEW pg_catalog.svv_external_tables AS+SELECT current_database()::text AS redshift_database_name, c.schema_name AS schemaname,+ t->>'tablename' AS tablename, t->>'tabletype' AS tabletype, t->>'location' AS location,+ t->>'input_format' AS input_format, t->>'output_format' AS output_format,+ t->>'serialization_lib' AS serialization_lib, t->>'serde_parameters' AS serde_parameters,+ (t->>'compressed')::integer AS compressed, t->>'parameters' AS parameters+FROM floci_internal.external_catalog c+JOIN pg_catalog.pg_namespace n ON n.nspname = c.schema_name+CROSS JOIN LATERAL jsonb_array_elements(c.payload->'tables') t+WHERE has_schema_privilege(n.oid, 'USAGE');++CREATE OR REPLACE VIEW pg_catalog.svv_external_columns AS+SELECT current_database()::text AS redshift_database_name, c.schema_name AS schemaname,+ t->>'tablename' AS tablename, t->>'columnname' AS columnname, t->>'external_type' AS external_type,+ (t->>'columnnum')::integer AS columnnum, (t->>'part_key')::integer AS part_key,+ t->>'is_nullable' AS is_nullable+FROM floci_internal.external_catalog c+JOIN pg_catalog.pg_namespace n ON n.nspname = c.schema_name+CROSS JOIN LATERAL jsonb_array_elements(c.payload->'columns') t+WHERE has_schema_privilege(n.oid, 'USAGE');++CREATE OR REPLACE VIEW pg_catalog.svv_external_partitions AS+SELECT c.schema_name AS schemaname, t->>'tablename' AS tablename, t->>'values' AS values,+ t->>'location' AS location, t->>'input_format' AS input_format, t->>'output_format' AS output_format,+ t->>'serialization_lib' AS serialization_lib, t->>'serde_parameters' AS serde_parameters,+ (t->>'compressed')::integer AS compressed, t->>'parameters' AS parameters+FROM floci_internal.external_catalog c+JOIN pg_catalog.pg_namespace n ON n.nspname = c.schema_name+CROSS JOIN LATERAL jsonb_array_elements(c.payload->'partitions') t+WHERE has_schema_privilege(n.oid, 'USAGE');+GRANT USAGE ON SCHEMA pg_catalog TO PUBLIC;GRANT ALL ON TABLE pg_catalog.stl_load_errors TO PUBLIC;GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO PUBLIC;diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.javaindex 988ddc863e..1dab9c0de8 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java@@ -1,9 +1,19 @@package io.github.hectorvent.floci.services.redshift;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.services.glue.GlueService;+import io.github.hectorvent.floci.services.glue.model.Database;import io.github.hectorvent.floci.services.redshift.model.Cluster;+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatement;+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatementParser;+import io.github.hectorvent.floci.services.redshift.spectrum.GlueTableBuilder;+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalCatalogRegistry;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.github.hectorvent.floci.services.s3.S3Service;import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.QuarkusTestProfile;+import io.quarkus.test.junit.TestProfile;import jakarta.inject.Inject;import org.awaitility.Awaitility;import org.junit.jupiter.api.AfterEach;@@ -18,18 +28,36 @@ import java.sql.DriverManager;import java.sql.ResultSet;import java.sql.SQLException;import java.sql.Statement;+import java.sql.Savepoint;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.sql.PreparedStatement;import java.time.Duration;import java.util.ArrayList;import java.util.List;import java.util.Objects;+import java.util.Map;+import org.postgresql.PGStatement;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTest+@TestProfile(RedshiftSpectrumIntegrationTest.SpectrumProfile.class)class RedshiftSpectrumIntegrationTest {+ public static class SpectrumProfile implements QuarkusTestProfile {+ @Override+ public Map<String, String> getConfigOverrides() {+ return Map.of("quarkus.http.test-port", "17866", "floci.base-url", "http://localhost:17866",+ "floci.docker.resource-namespace", "spectrum-relational-tests");+ }+ }+@InjectRedshiftService redshiftService;@@ -39,6 +67,12 @@ class RedshiftSpectrumIntegrationTest {@InjectIamService iamService;+ @Inject+ GlueService glueService;++ @Inject+ ExternalCatalogRegistry externalRegistry;+private static final String ROLE_NAME = "SpectrumItRole";private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/" + ROLE_NAME;private static final String TRUST_POLICY = """@@ -47,6 +81,7 @@ class RedshiftSpectrumIntegrationTest {private String clusterId;private String bucket;+ private String glueDatabase;@BeforeAllstatic void requireDocker() {@@ -86,6 +121,200 @@ class RedshiftSpectrumIntegrationTest {s3Service.deleteObject(bucket, "invalid/part-1.csv");s3Service.deleteBucket(bucket);}+ if (glueDatabase != null) {+ glueService.deleteDatabase(glueDatabase);+ }+ }++ @Test+ @Timeout(120)+ void joinsExternalAndInternalTablesWithAggregation() throws Exception {+ bucket = "spectrum-relational-" + System.nanoTime();+ s3Service.createBucket(bucket, "us-east-1");+ s3Service.putObject(bucket, "events/part-1.csv", "1,10\n1,5\n2,7\n".getBytes(StandardCharsets.UTF_8),+ "text/csv", null);+ glueDatabase = "spectrum_relational_" + System.nanoTime();+ Database database = new Database();+ database.setName(glueDatabase);+ glueService.createDatabase(database);+ clusterId = "it-relational-" + System.nanoTime();+ Cluster cluster = createClusterWithRole(clusterId);+ try (Connection connection = waitForConnection(cluster, "admin", "Secret123");+ Statement statement = connection.createStatement()) {+ statement.execute("CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE '" + glueDatabase+ + "' IAM_ROLE '" + ROLE_ARN + "'");+ String externalDdl = "CREATE EXTERNAL TABLE lake.events (id INTEGER, amount INTEGER) "+ + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE LOCATION 's3://"+ + bucket + "/events/'";+ ExternalStatement.CreateTable external = (ExternalStatement.CreateTable)+ new ExternalStatementParser().parse(externalDdl).orElseThrow();+ glueService.createTable(glueDatabase, GlueTableBuilder.toGlueTable(external));+ statement.execute("CREATE TABLE customers (id INTEGER, country VARCHAR(2))");+ statement.execute("INSERT INTO customers VALUES (1, 'VN'), (2, 'US')");+ try (PreparedStatement ddl = connection.prepareStatement("CREATE EXTERNAL SCHEMA rolledback_lake "+ + "FROM DATA CATALOG DATABASE '" + glueDatabase + "' IAM_ROLE '" + ROLE_ARN + "'")) {+ ddl.unwrap(PGStatement.class).setPrepareThreshold(1);+ assertEquals(0, ddl.getParameterMetaData().getParameterCount());+ connection.setAutoCommit(false);+ try {+ SQLException rejected = assertThrows(SQLException.class, ddl::execute);+ assertEquals("0A000", rejected.getSQLState());+ } finally {+ connection.rollback();+ connection.setAutoCommit(true);+ }+ assertTrue(externalRegistry.find("000000000000", "000000000000:" + clusterId,+ "dev", "rolledback_lake").isEmpty());+ }+ connection.setAutoCommit(false);+ try {+ SQLException rejected = assertThrows(SQLException.class, () -> statement.execute(+ externalDdl.replace("lake.events", "lake.in_transaction")));+ assertEquals("25001", rejected.getSQLState());+ assertTrue(rejected.getMessage().contains("cannot run inside a transaction block"));+ } finally {+ connection.rollback();+ connection.setAutoCommit(true);+ }+ assertThrows(AwsException.class, () -> glueService.getTable(glueDatabase, "in_transaction"));+ try (PreparedStatement query = connection.prepareStatement("SELECT c.country, SUM(e.amount) AS total "+ + "FROM lake.events e JOIN customers c ON c.id=e.id WHERE e.amount > ? "+ + "GROUP BY c.country ORDER BY total DESC")) {+ query.setInt(1, 0);+ try (ResultSet rows = query.executeQuery()) {+ assertTrue(rows.next());+ assertEquals("VN", rows.getString(1));+ assertEquals(15L, rows.getLong(2));+ assertTrue(rows.next());+ assertEquals("US", rows.getString(1));+ assertEquals(7L, rows.getLong(2));+ assertTrue(!rows.next());+ }+ query.unwrap(PGStatement.class).setPrepareThreshold(1);+ statement.execute("CREATE VIEW event_amounts AS SELECT amount FROM lake.events");+ s3Service.putObject(bucket, "events/part-1.csv", "1,20\n2,7\n".getBytes(StandardCharsets.UTF_8),+ "text/csv", null);+ try (ResultSet rows = query.executeQuery()) {+ assertTrue(rows.next());+ assertEquals(20L, rows.getLong(2));+ }+ s3Service.putObject(bucket, "events/part-1.csv", "1,30\n2,7\n".getBytes(StandardCharsets.UTF_8),+ "text/csv", null);+ try (ResultSet rows = query.executeQuery()) {+ assertTrue(rows.next());+ assertEquals(30L, rows.getLong(2));+ }+ }+ try (ResultSet rows = statement.executeQuery("SELECT SUM(a.amount) FROM lake.events a "+ + "JOIN lake.events b ON a.id=b.id HAVING SUM(a.amount)>10 ORDER BY 1 LIMIT 1")) {+ assertTrue(rows.next());+ assertEquals(37L, rows.getLong(1));+ }+ try (ResultSet rows = statement.executeQuery("SELECT SUM(amount) FROM event_amounts")) {+ assertTrue(rows.next());+ assertEquals(37L, rows.getLong(1));+ }+ connection.setAutoCommit(false);+ s3Service.putObject(bucket, "events/part-1.csv", "1,35\n2,7\n".getBytes(StandardCharsets.UTF_8),+ "text/csv", null);+ try (PreparedStatement cursor = connection.prepareStatement("SELECT id, amount FROM lake.events ORDER BY id")) {+ cursor.setFetchSize(1);+ try (ResultSet rows = cursor.executeQuery()) {+ assertTrue(rows.next());+ assertEquals(1, rows.getInt(1));+ try (PreparedStatement another = connection.prepareStatement("SELECT SUM(amount) FROM lake.events");+ ResultSet sum = another.executeQuery()) {+ assertTrue(sum.next());+ assertEquals(42L, sum.getLong(1));+ }+ assertTrue(rows.next());+ assertEquals(2, rows.getInt(1));+ assertTrue(!rows.next());+ }+ }+ Savepoint beforeReload = connection.setSavepoint();+ s3Service.putObject(bucket, "events/part-1.csv", "1,40\n2,7\n".getBytes(StandardCharsets.UTF_8),+ "text/csv", null);+ try (ResultSet rows = statement.executeQuery("SELECT SUM(amount) FROM lake.events")) {+ assertTrue(rows.next());+ assertEquals(47L, rows.getLong(1));+ }+ connection.rollback(beforeReload);+ try (ResultSet rows = statement.executeQuery("SELECT SUM(amount) FROM lake.events")) {+ assertTrue(rows.next());+ assertEquals(47L, rows.getLong(1));+ }+ connection.rollback();+ connection.setAutoCommit(true);+ try (ResultSet rows = statement.executeQuery("SELECT SUM(amount) FROM lake.events")) {+ assertTrue(rows.next());+ assertEquals(47L, rows.getLong(1));+ }+ s3Service.putObject(bucket, "events/part-1.csv", "1,30\n2,7\n".getBytes(StandardCharsets.UTF_8),+ "text/csv", null);+ RestAssuredJsonUtils.configureAwsContentTypes();+ String queryId = RestAssuredJsonUtils.awsAction("RedshiftData", "ExecuteStatement", """+ {"ClusterIdentifier":"%s","DbUser":"admin","Database":"dev",+ "Sql":"SELECT SUM(amount) AS total FROM lake.events"}+ """.formatted(clusterId)).then().statusCode(200).extract().path("Id");+ assertEquals("FINISHED", RestAssuredJsonUtils.awsAction("RedshiftData", "DescribeStatement",+ "{\"Id\":\"" + queryId + "\"}").then().statusCode(200).extract().path("Status"));+ Number total = RestAssuredJsonUtils.awsAction("RedshiftData", "GetStatementResult",+ "{\"Id\":\"" + queryId + "\"}").then().statusCode(200).extract().path("Records[0][0].longValue");+ assertEquals(37L, total.longValue());+ iamService.updateAssumeRolePolicy(ROLE_NAME, """+ {"Version":"2012-10-17","Statement":[{"Effect":"Deny",+ "Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}""");+ try {+ SQLException denied = assertThrows(SQLException.class,+ () -> statement.executeQuery("SELECT SUM(amount) FROM lake.events"));+ assertEquals("42501", denied.getSQLState());+ } finally {+ iamService.updateAssumeRolePolicy(ROLE_NAME, TRUST_POLICY);+ }+ statement.execute("CREATE USER spectrum_reader PASSWORD 'Password123'");+ statement.execute("GRANT USAGE ON SCHEMA lake TO spectrum_reader");+ statement.execute("GRANT SELECT ON lake.events TO spectrum_reader");+ try (Connection reader = DriverManager.getConnection("jdbc:postgresql://127.0.0.1:"+ + cluster.getEndpoint().getPort() + "/dev?socketTimeout=20", "spectrum_reader", "Password123");+ Statement readerQuery = reader.createStatement();+ ResultSet rows = readerQuery.executeQuery("SELECT SUM(amount) FROM lake.events")) {+ assertTrue(rows.next());+ assertEquals(37L, rows.getLong(1));+ }+ s3Service.putObject(bucket, "events/part-1.csv", "1,40\n2,7\n".getBytes(StandardCharsets.UTF_8),+ "text/csv", null);+ CountDownLatch ready = new CountDownLatch(2);+ CountDownLatch start = new CountDownLatch(1);+ try (ExecutorService queries = Executors.newVirtualThreadPerTaskExecutor()) {+ Future<Long> first = queries.submit(() -> concurrentTotal(cluster, ready, start));+ Future<Long> second = queries.submit(() -> concurrentTotal(cluster, ready, start));+ assertTrue(ready.await(20, TimeUnit.SECONDS));+ start.countDown();+ assertEquals(47L, first.get(30, TimeUnit.SECONDS));+ assertEquals(47L, second.get(30, TimeUnit.SECONDS));+ }+ s3Service.putObject(bucket, "events/part-1.csv", "bad,40\n".getBytes(StandardCharsets.UTF_8),+ "text/csv", null);+ assertThrows(SQLException.class, () -> statement.executeQuery("SELECT SUM(amount) FROM lake.events"));+ s3Service.putObject(bucket, "events/part-1.csv", "1,40\n2,7\n".getBytes(StandardCharsets.UTF_8),+ "text/csv", null);+ try (ResultSet rows = statement.executeQuery("SELECT SUM(amount) FROM lake.events")) {+ assertTrue(rows.next());+ assertEquals(47L, rows.getLong(1));+ }+ glueService.deleteTable(glueDatabase, "events");+ SQLException missing = assertThrows(SQLException.class,+ () -> statement.executeQuery("SELECT * FROM lake.events"));+ assertEquals("42P01", missing.getSQLState());+ try (ResultSet rows = statement.executeQuery("SELECT 1")) {+ assertTrue(rows.next());+ }+ }+ String deleted = clusterId;+ redshiftService.deleteCluster(deleted);+ clusterId = null;+ assertTrue(externalRegistry.list("000000000000", "000000000000:" + deleted, "dev").isEmpty());}@Test@@ -214,4 +443,16 @@ class RedshiftSpectrumIntegrationTest {.pollInterval(Duration.ofMillis(500)).ignoreExceptions().until(() -> DriverManager.getConnection(url, username, password), Objects::nonNull);}++ private static long concurrentTotal(Cluster cluster, CountDownLatch ready, CountDownLatch start) throws Exception {+ try (Connection connection = waitForConnection(cluster, "admin", "Secret123");+ PreparedStatement query = connection.prepareStatement("SELECT SUM(amount) FROM lake.events")) {+ ready.countDown();+ assertTrue(start.await(20, TimeUnit.SECONDS));+ try (ResultSet rows = query.executeQuery()) {+ assertTrue(rows.next());+ return rows.getLong(1);+ }+ }+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.javaindex 3ef223e0f2..ec81fa4257 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java@@ -14,6 +14,28 @@ class BackendResponseCoordinatorTest {private static final byte[] EMPTY_BODY = new byte[0];+ @Test+ void beginUpdatesTransactionStateBeforeSync() {+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());+ coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);+ coordinator.onBackendFrame('C', new byte[]{'B', 'E', 'G', 'I', 'N', 0});+ assertEquals('T', coordinator.lastReadyStatus());+ coordinator.onBackendFrame('C', new byte[]{'R', 'O', 'L', 'L', 'B', 'A', 'C', 'K', 0});+ assertEquals('T', coordinator.lastReadyStatus());+ coordinator.onBackendFrame('Z', new byte[]{'I'});+ assertEquals('I', coordinator.lastReadyStatus());+ }++ @Test+ void commitLeavesTransactionBeforeSync() {+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());+ coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);+ coordinator.onBackendFrame('C', new byte[]{'B', 'E', 'G', 'I', 'N', 0});+ coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);+ coordinator.onBackendFrame('C', new byte[]{'C', 'O', 'M', 'M', 'I', 'T', 0});+ assertEquals('I', coordinator.lastReadyStatus());+ }+@Testvoid parseBindAndDescribeCompleteBeforeExecuteBecomesReady() throws InterruptedException {ExtendedQuerySession session = new ExtendedQuerySession();diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.javaindex 4cb26e418c..83102a0d7f 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.redshift.proxy;import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumInterceptor;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import org.jboss.logging.Logger;@@ -11,9 +12,11 @@ import org.mockito.Mockito;import java.io.ByteArrayOutputStream;import java.io.IOException;+import java.io.InputStream;import java.io.OutputStream;import java.net.ServerSocket;import java.net.Socket;+import java.nio.ByteBuffer;import java.nio.charset.StandardCharsets;import java.util.List;import java.util.concurrent.atomic.AtomicReference;@@ -26,6 +29,18 @@ import static org.junit.jupiter.api.Assertions.assertTrue;class RedshiftInterceptingBridgeTest {+ @Test+ void tracksBeginCommandAcrossFragmentedBackendFrames() {+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());+ RedshiftInterceptingBridge.WireFrameTracker tracker =+ new RedshiftInterceptingBridge.WireFrameTracker(coordinator::onBackendFrame);+ byte[] begin = new byte[]{'C', 0, 0, 0, 10, 'B', 'E', 'G', 'I', 'N', 0};+ tracker.consume(begin, 0, 7);+ assertEquals('I', coordinator.lastReadyStatus());+ tracker.consume(begin, 7, 4);+ assertEquals('T', coordinator.lastReadyStatus());+ }+private static final Logger LOG = Logger.getLogger(RedshiftInterceptingBridgeTest.class);private ServerSocket clientListener;@@ -39,6 +54,10 @@ class RedshiftInterceptingBridgeTest {private S3Service s3Stub;private void startBridge() throws IOException {+ startBridge(null);+ }++ private void startBridge(SpectrumInterceptor spectrumInterceptor) throws IOException {clientListener = new ServerSocket(0);testClientEnd = new Socket("localhost", clientListener.getLocalPort());bridgeClientEnd = clientListener.accept();@@ -49,7 +68,8 @@ class RedshiftInterceptingBridgeTest {s3Stub = Mockito.mock(S3Service.class);RedshiftInterceptingBridge bridge = new RedshiftInterceptingBridge(- bridgeClientEnd, bridgeBackendEnd, s3Stub, Mockito.mock(IamService.class));+ bridgeClientEnd, bridgeBackendEnd, s3Stub, Mockito.mock(IamService.class), "000000000000", List.of(),+ spectrumInterceptor, "000000000000:c", "dev");bridgeThread = Thread.ofVirtual().name("bridge-under-test").start(bridge::run);}@@ -306,6 +326,39 @@ class RedshiftInterceptingBridgeTest {return result;}+ @Test+ @Timeout(20)+ void locallyHandledQueryInsideTransactionKeepsTheTransactionStatus() throws Exception {+ SpectrumInterceptor interceptor = Mockito.mock(SpectrumInterceptor.class);+ Mockito.when(interceptor.intercept(Mockito.anyString(), Mockito.any(), Mockito.any(Socket.class)))+ .thenReturn(new SpectrumInterceptor.Decision.Forward(), new SpectrumInterceptor.Decision.Handled());+ startBridge(interceptor);+ InputStream fromBridge = testClientEnd.getInputStream();++ testClientEnd.getOutputStream().write(PostgresWireDecoder.encodeQuery("BEGIN"));+ testClientEnd.getOutputStream().flush();+ assertEquals('Q', nextForwarded().type());+ writeBackendFrame('C', cString("BEGIN"));+ writeBackendFrame('Z', new byte[]{'T'});+ assertEquals('C', fromBridge.read());+ skipFrameBody(fromBridge);+ assertEquals('Z', fromBridge.read());+ skipFrameBody(fromBridge);++ testClientEnd.getOutputStream().write(PostgresWireDecoder.encodeQuery("CREATE EXTERNAL TABLE lake.t (id INT)"));+ testClientEnd.getOutputStream().flush();+ assertEquals('C', fromBridge.read());+ skipFrameBody(fromBridge);+ assertEquals('Z', fromBridge.read());+ fromBridge.readNBytes(4);+ assertEquals('T', fromBridge.read());+ }++ private static void skipFrameBody(InputStream in) throws IOException {+ int length = ByteBuffer.wrap(in.readNBytes(4)).getInt();+ in.readNBytes(length - 4);+ }+@Testvoid pumpsBackendBytesToTheClientUnchanged() throws IOException {startBridge();diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.javaindex e2662a87b8..f42ba4594e 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java@@ -39,6 +39,7 @@ import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.not;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertEquals;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyInt;import static org.mockito.ArgumentMatchers.anyString;@@ -69,6 +70,12 @@ class ExternalTableMaterializerTest {private static final class RecordingBackend implements BackendSql {private final List<String> statements = new ArrayList<>();+ private boolean publicationAllowed = true;++ @Override+ public boolean permitsCachePublication() {+ return publicationAllowed;+ }@Overridepublic void execute(String sql) {@@ -115,6 +122,37 @@ class ExternalTableMaterializerTest {.thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));}+ @Test+ void confirmedExtendedCommitPublishesDefinitionsAndFingerprint() {+ when(glue.getTable("lake", "events")).thenReturn(csvTable());+ backend.publicationAllowed = false;+ materializer.ensureCurrent(backend, session(false), BINDING, "events");+ materializer.finishCycle(backend, true);+ RecordingBackend other = new RecordingBackend();+ assertEquals(ExternalTableMaterializer.Outcome.CURRENT,+ materializer.ensureCurrent(other, session(false), BINDING, "events"));+ forceReload();+ materializer.ensureCurrent(other, session(false), BINDING, "events");+ assertThat(other.statements.getLast(), containsString("CREATE TABLE IF NOT EXISTS"));+ }++ @Test+ void extendedLoadsAreReusableOnlyWithinTheirOwnCycle() {+ when(glue.getTable("lake", "events")).thenReturn(csvTable());+ backend.publicationAllowed = false;+ assertEquals(ExternalTableMaterializer.Outcome.LOADED,+ materializer.ensureCurrent(backend, session(false), BINDING, "events"));+ assertEquals(ExternalTableMaterializer.Outcome.CURRENT,+ materializer.ensureCurrent(backend, session(false), BINDING, "events"));+ RecordingBackend other = new RecordingBackend();+ other.publicationAllowed = false;+ assertEquals(ExternalTableMaterializer.Outcome.LOADED,+ materializer.ensureCurrent(other, session(false), BINDING, "events"));+ materializer.finishCycle(backend);+ assertEquals(ExternalTableMaterializer.Outcome.LOADED,+ materializer.ensureCurrent(backend, session(false), BINDING, "events"));+ }+@Testvoid loadsThroughAStagingTableAndCachesUnchangedData() {Table table = csvTable();diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.javaindex c0614bb9d4..976e03bcb1 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java@@ -19,6 +19,78 @@ import static org.junit.jupiter.api.Assertions.assertThrows;class PostgresBackendSessionTest {+ @Test+ void extendedFailureDoesNotWaitForIgnoredCleanupBeforeClientSync() throws Exception {+ try (ServerSocket listener = new ServerSocket(0);+ Socket backend = new Socket("localhost", listener.getLocalPort())) {+ ExecutorService executor = Executors.newSingleThreadExecutor();+ Future<String> names = executor.submit(() -> {+ try (Socket socket = listener.accept()) {+ String first = "";+ Frame frame;+ do {+ frame = readFrame(socket.getInputStream());+ if (frame.type() == 'P') {+ first = new String(frame.body(), StandardCharsets.UTF_8).split("\0", 2)[0];+ }+ } while (frame.type() != 'H');+ send(socket.getOutputStream(), 'E', new byte[]{'C', '4', '2', 'P', '0', '1', 0,+ 'M', 'm', 'i', 's', 's', 'i', 'n', 'g', 0, 0});+ String second = "";+ do {+ frame = readFrame(socket.getInputStream());+ if (frame.type() == 'P') {+ second = new String(frame.body(), StandardCharsets.UTF_8).split("\0", 2)[0];+ }+ } while (frame.type() != 'H');+ send(socket.getOutputStream(), 'C', "SELECT 1\0".getBytes(StandardCharsets.UTF_8));+ send(socket.getOutputStream(), '3', new byte[0]);+ send(socket.getOutputStream(), '3', new byte[0]);+ return first.equals(second) ? "reused" : "distinct";+ }+ });+ PostgresExtendedBackendSession sql = new PostgresExtendedBackendSession(backend);+ assertThrows(SpectrumReadException.class, () -> sql.execute("SELECT missing"));+ SpectrumReadException cleanup = assertThrows(SpectrumReadException.class,+ () -> sql.execute("DROP TABLE t"));+ assertEquals("25P02", cleanup.sqlState());+ sql.onSync();+ sql.execute("SELECT 1");+ assertEquals("distinct", names.get());+ executor.shutdownNow();+ }+ }++ @Test+ void extendedPreparationClosesOnlyItsNamedObjectsWithoutSyncOrSimpleQuery() throws Exception {+ try (ServerSocket listener = new ServerSocket(0);+ Socket backend = new Socket("localhost", listener.getLocalPort())) {+ ExecutorService executor = Executors.newSingleThreadExecutor();+ Future<String> messages = executor.submit(() -> {+ try (Socket socket = listener.accept()) {+ StringBuilder types = new StringBuilder();+ Frame frame;+ do {+ frame = readFrame(socket.getInputStream());+ types.append(frame.type());+ if (frame.type() == 'P') {+ assertEquals('f', (char) frame.body()[0]);+ }+ } while (frame.type() != 'H');+ send(socket.getOutputStream(), '1', new byte[0]);+ send(socket.getOutputStream(), '2', new byte[0]);+ send(socket.getOutputStream(), 'C', "DO\0".getBytes(StandardCharsets.UTF_8));+ send(socket.getOutputStream(), '3', new byte[0]);+ send(socket.getOutputStream(), '3', new byte[0]);+ return types.toString();+ }+ });+ new PostgresExtendedBackendSession(backend).execute("CREATE TABLE t(id integer); INSERT INTO t VALUES (1)");+ assertEquals("PBECCH", messages.get());+ executor.shutdownNow();+ }+ }+@Testvoid copyReadFailureDrainsCopyFailResponseWithoutReplacingOriginalCause() throws Exception {try (ServerSocket listener = new ServerSocket(0);@@ -188,4 +260,13 @@ class PostgresBackendSessionTest {private record Frame(char type, byte[] body) {}++ @Test+ void preparationFailsFastAfterTheClientsOwnBackendError() {+ PostgresExtendedBackendSession session = new PostgresExtendedBackendSession(null);+ session.onBackendError();+ SpectrumReadException error = assertThrows(SpectrumReadException.class, () -> session.execute("CREATE TABLE t(id INT)"));+ assertEquals("25P02", error.sqlState());+ session.onSync();+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparationTest.javanew file mode 100644index 0000000000..7a2c5dacba--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparationTest.java@@ -0,0 +1,167 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import io.github.hectorvent.floci.services.glue.GlueService;+import io.github.hectorvent.floci.core.common.AwsException;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Optional;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.*;++class SpectrumQueryPreparationTest {+ private static final SpectrumSession SESSION = new SpectrumSession("000000000000", "000000000000:c",+ "dev", List.of(), false);+ private static final ExternalSchemaBinding BINDING = new ExternalSchemaBinding(SESSION.accountId(),+ SESSION.clusterKey(), "dev", "lake", "analytics", "arn:aws:iam::000000000000:role/R");+ private ExternalCatalogRegistry registry;+ private ExternalTableMaterializer materializer;+ private BackendSql backend;+ private SpectrumQueryPreparation preparation;+ private GlueService glue;++ @BeforeEach+ void setUp() {+ registry = mock(ExternalCatalogRegistry.class);+ materializer = mock(ExternalTableMaterializer.class);+ backend = mock(BackendSql.class);+ when(registry.list(SESSION.accountId(), SESSION.clusterKey(), "dev")).thenReturn(List.of(BINDING));+ when(materializer.ensureCurrent(any(), any(), any(), anyString()))+ .thenReturn(ExternalTableMaterializer.Outcome.CURRENT);+ glue = mock(GlueService.class);+ preparation = new SpectrumQueryPreparation(registry, materializer, new ExternalStatementParser(),+ glue, mock(ExternalMetadataWriter.class));+ }++ @Test+ void glueDdlFailureUsesSqlErrorInsteadOfEscapingTheWireHandler() {+ doThrow(new AwsException("AlreadyExistsException", "table already exists", 400))+ .when(glue).createTable(anyString(), any());+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () -> preparation.prepare(+ "CREATE EXTERNAL TABLE lake.sales (id INTEGER) STORED AS TEXTFILE LOCATION 's3://bucket/'",+ SESSION, backend));+ assertEquals("42P07", error.sqlState());+ }++ @Test+ void transactionalSchemaCreateFailsBeforeChangingEitherStore() {+ SpectrumSession transaction = new SpectrumSession(SESSION.accountId(), SESSION.clusterKey(),+ SESSION.databaseName(), List.of(), true);+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () -> preparation.prepare(+ "CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE 'analytics' IAM_ROLE '"+ + BINDING.iamRoleArn() + "'", transaction, backend));+ assertEquals("0A000", error.sqlState());+ verifyNoInteractions(glue, backend);+ verify(registry, never()).bind(any());+ }++ @Test+ void preparesDistinctExternalReferences() {+ assertFalse(preparation.prepare("SELECT a.id, SUM(b.amount) FROM lake.sales a JOIN lake.sales b "+ + "ON a.id=b.id JOIN public.customers c ON c.id=a.id GROUP BY a.id ORDER BY a.id", SESSION, backend));+ verify(materializer).ensureCurrent(backend, SESSION, BINDING, "sales");+ }++ @Test+ void leavesInternalSqlUntouched() {+ assertFalse(preparation.prepare("SELECT 'lake.sales' FROM public.customers lake", SESSION, backend));+ verifyNoInteractions(materializer, backend);+ }++ @Test+ void rejectsExternalMutationInCte() {+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () -> preparation.prepare(+ "WITH changed AS (DELETE FROM lake.sales RETURNING *) SELECT * FROM changed", SESSION, backend));+ assertEquals("0A000", error.sqlState());+ verifyNoInteractions(materializer, backend);+ }++ @Test+ void missingGlueTableDoesNotUseCachedRelation() {+ when(materializer.ensureCurrent(backend, SESSION, BINDING, "sales"))+ .thenReturn(ExternalTableMaterializer.Outcome.NOT_EXTERNAL);+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () ->+ preparation.prepare("SELECT * FROM lake.sales", SESSION, backend));+ assertEquals("42P01", error.sqlState());+ }++ @Test+ void failedSchemaCreateDoesNotPublishBinding() {+ when(registry.find(anyString(), anyString(), anyString(), anyString())).thenReturn(Optional.empty());+ doThrow(new SpectrumReadException("58030", "backend unavailable")).when(backend).execute(anyString());+ assertThrows(SpectrumReadException.class, () -> preparation.prepare(+ "CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE 'analytics' IAM_ROLE '"+ + BINDING.iamRoleArn() + "'", SESSION, backend));+ verify(registry, never()).bind(any());+ }++ private static final String CREATE_LAKE = "CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE 'analytics' IAM_ROLE '"+ + BINDING.iamRoleArn() + "'";++ @Test+ void schemaCreatedInClientTransactionIsBoundOnlyWhenItCommits() {+ when(registry.find(anyString(), anyString(), anyString(), anyString())).thenReturn(Optional.empty());+ when(backend.permitsCachePublication()).thenReturn(false);+ assertTrue(preparation.prepare(CREATE_LAKE, SESSION, backend));+ verify(registry, never()).bind(any());+ preparation.finishCycle(backend, true);+ verify(registry).bind(any());+ }++ @Test+ void schemaCreatedInRolledBackTransactionIsNeverBound() {+ when(registry.find(anyString(), anyString(), anyString(), anyString())).thenReturn(Optional.empty());+ when(backend.permitsCachePublication()).thenReturn(false);+ assertTrue(preparation.prepare(CREATE_LAKE, SESSION, backend));+ preparation.finishCycle(backend);+ preparation.finishCycle(backend, true);+ verify(registry, never()).bind(any());+ }++ private static final SpectrumSession TRANSACTION = new SpectrumSession(SESSION.accountId(), SESSION.clusterKey(),+ "dev", List.of(), true);+ private static final String CREATE_SALES =+ "CREATE EXTERNAL TABLE lake.sales (id INTEGER) STORED AS TEXTFILE LOCATION 's3://bucket/'";++ @Test+ void transactionalTableCreateIsRejectedBeforeChangingEitherStore() {+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,+ () -> preparation.prepare(CREATE_SALES, TRANSACTION, backend));+ assertEquals("25001", error.sqlState());+ assertEquals("CREATE EXTERNAL TABLE cannot run inside a transaction block", error.getMessage());+ verifyNoInteractions(glue, backend);+ }++ @Test+ void transactionalTableCreateInUnboundSchemaIsRejectedAndRoutedToPreparation() {+ String legacy = "CREATE EXTERNAL TABLE legacy.sales (id INTEGER) STORED AS TEXTFILE LOCATION 's3://bucket/'";+ assertTrue(preparation.handlesDdl(legacy, TRANSACTION));+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,+ () -> preparation.prepare(legacy, TRANSACTION, backend));+ assertEquals("25001", error.sqlState());+ verifyNoInteractions(glue, backend);+ }++ @Test+ void tableCreatedOutsideTransactionIsStoredInGlue() {+ assertTrue(preparation.prepare(CREATE_SALES, SESSION, backend));+ verify(glue).createTable(eq("analytics"), any());+ }++ @Test+ void discardingLoadsOnlyDropsPendingLoads() {+ preparation.discardPendingLoads(backend);+ verify(materializer).finishCycle(backend);+ verifyNoInteractions(glue);+ }++ @Test+ void schemaCreatedOnAutocommitBackendIsBoundImmediately() {+ when(registry.find(anyString(), anyString(), anyString(), anyString())).thenReturn(Optional.empty());+ when(backend.permitsCachePublication()).thenReturn(true);+ assertTrue(preparation.prepare(CREATE_LAKE, SESSION, backend));+ verify(registry).bind(any());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSqlTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSqlTest.javanew file mode 100644index 0000000000..1827ecc1a3--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSqlTest.java@@ -0,0 +1,47 @@+package io.github.hectorvent.floci.services.redshiftdata;++import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumReadException;+import org.junit.jupiter.api.Test;+import org.postgresql.PGConnection;+import org.postgresql.copy.CopyManager;++import java.io.ByteArrayInputStream;+import java.io.InputStream;+import java.sql.Connection;+import java.sql.SQLException;+import java.sql.Statement;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.*;++class RedshiftDataBackendSqlTest {+ @Test+ void usesCallerConnectionWithoutCommittingOrClosingIt() throws Exception {+ Connection connection = mock(Connection.class);+ Statement statement = mock(Statement.class);+ PGConnection pg = mock(PGConnection.class);+ CopyManager copy = mock(CopyManager.class);+ when(connection.createStatement()).thenReturn(statement);+ when(connection.unwrap(PGConnection.class)).thenReturn(pg);+ when(pg.getCopyAPI()).thenReturn(copy);+ InputStream data = new ByteArrayInputStream(new byte[]{1});+ when(copy.copyIn("COPY t FROM STDIN", data)).thenReturn(2L);+ RedshiftDataBackendSql backend = new RedshiftDataBackendSql(connection);+ backend.execute("CREATE TABLE t(id INTEGER)");+ assertEquals(2L, backend.copyIn("COPY t FROM STDIN", data));+ verify(statement).execute("CREATE TABLE t(id INTEGER)");+ verify(statement).close();+ verify(connection, never()).commit();+ verify(connection, never()).rollback();+ verify(connection, never()).close();+ }++ @Test+ void preservesSqlStateOnFailure() throws Exception {+ Connection connection = mock(Connection.class);+ when(connection.createStatement()).thenThrow(new SQLException("denied", "42501"));+ SpectrumReadException error = assertThrows(SpectrumReadException.class,+ () -> new RedshiftDataBackendSql(connection).execute("SELECT 1"));+ assertEquals("42501", error.sqlState());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolverTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolverTest.javaindex 92204ed42c..4784c5998a 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolverTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolverTest.java@@ -69,7 +69,8 @@ class RedshiftDataResourceResolverTest {private static RedshiftServerlessService.WorkgroupTarget workgroupTarget() {return new RedshiftServerlessService.WorkgroupTarget("arn:aws:redshift-serverless:us-east-1:" + ACCOUNT + ":workgroup/wg-id", "wg-1",- "127.0.0.1", 55433, "analytics", "root", "Secret123", "root");+ "127.0.0.1", 55433, "analytics", "root", "Secret123", "root",+ List.of("arn:aws:iam::" + ACCOUNT + ":role/Spectrum"));}@Test@@ -88,6 +89,10 @@ class RedshiftDataResourceResolverTest {assertEquals("analytics", target.database());assertEquals("root", target.user());assertEquals("Secret123", target.password());+ assertEquals(ACCOUNT, target.spectrum().accountId());+ assertEquals(ACCOUNT + ":serverless_us-east-1_wg-1", target.spectrum().clusterKey());+ assertEquals("analytics", target.spectrum().databaseName());+ assertEquals(List.of("arn:aws:iam::" + ACCOUNT + ":role/Spectrum"), target.spectrum().iamRoleArns());}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.javaindex 4c5354da94..0ee9b680d1 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.java@@ -5,6 +5,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumQueryPreparation;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSession;+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumReadException;import org.h2.Driver;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;@@ -14,6 +17,7 @@ import java.sql.ResultSetMetaData;import java.sql.SQLException;import java.time.Clock;import java.util.UUID;+import java.util.List;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -22,6 +26,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.when;+import static org.mockito.Mockito.doThrow;+import static org.mockito.Mockito.verify;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.ArgumentMatchers.argThat;class RedshiftDataServiceTest {@@ -34,6 +42,7 @@ class RedshiftDataServiceTest {private final ObjectMapper om = new ObjectMapper();private String jdbcUrl;private RedshiftDataService service;+ private SpectrumQueryPreparation preparation;@BeforeEachvoid setUp() throws Exception {@@ -41,13 +50,15 @@ class RedshiftDataServiceTest {RedshiftDataResourceResolver resolver = mock(RedshiftDataResourceResolver.class);when(resolver.resolve(any(), any())).thenReturn(new RedshiftDataResourceResolver.DatabaseTarget(- "arn:aws:redshift:us-east-1:000000000000:cluster:wh", "127.0.0.1", 5439, "dev", "admin", "x"));+ "arn:aws:redshift:us-east-1:000000000000:cluster:wh", "127.0.0.1", 5439, "dev", "admin", "x",+ new SpectrumSession("000000000000", "000000000000:wh", "dev", List.of(), false)));RedshiftDataConnectionFactory factory = mock(RedshiftDataConnectionFactory.class);when(factory.open(any())).thenAnswer(inv -> DriverManager.getConnection(jdbcUrl, "sa", ""));+ preparation = mock(SpectrumQueryPreparation.class);service = new RedshiftDataService(resolver, factory,- new RedshiftDataStatementStore(24, Clock.systemUTC()), om);+ new RedshiftDataStatementStore(24, Clock.systemUTC()), om, preparation);}private ObjectNode req(String sql) {@@ -63,6 +74,24 @@ class RedshiftDataServiceTest {return om.createObjectNode().put("Id", id);}+ @Test+ void preparationFailureIsStoredAsFailedStatement() {+ doThrow(new SpectrumReadException("42501", "S3 access denied")).when(preparation)+ .prepare(eq("SELECT * FROM lake.events"), any(), any());+ ObjectNode executed = service.executeStatement(req("SELECT * FROM lake.events"), REGION);+ ObjectNode described = service.describeStatement(idOf(executed.path("Id").asText()));+ assertEquals("FAILED", described.path("Status").asText());+ assertEquals("S3 access denied", described.path("Error").asText());+ }++ @Test+ void batchPreparationUsesTransactionContext() {+ ObjectNode request = req("unused");+ request.putArray("Sqls").add("SELECT 1");+ service.batchExecuteStatement(request, REGION);+ verify(preparation).prepare(eq("SELECT 1"), argThat(SpectrumSession::inTransaction), any());+ }+@Testvoid aWorkgroupStatementNamesItsWorkgroupAndOmitsTheClusterIdentifier() {ObjectNode request = om.createObjectNode();