Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(redshift): add relational queries for Redshift Spectrum (#5095)

Vũ Thế Huy950b6fe774cc
diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftRelationalSpectrumTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftRelationalSpectrumTest.java
new file mode 100644
index 0000000000..ba459377c1
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftRelationalSpectrumTest.java
@@ -0,0 +1,143 @@
+package io.github.hectorvent.floci.compat;
+
+import com.floci.test.TestFixtures;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.glue.GlueClient;
+import software.amazon.awssdk.services.glue.model.DatabaseInput;
+import software.amazon.awssdk.services.iam.IamClient;
+import software.amazon.awssdk.services.redshift.RedshiftClient;
+import software.amazon.awssdk.services.redshift.model.Cluster;
+import software.amazon.awssdk.services.redshiftdata.RedshiftDataClient;
+import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementResponse;
+import software.amazon.awssdk.services.redshiftdata.model.GetStatementResultResponse;
+import software.amazon.awssdk.services.s3.S3Client;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.time.Instant;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class RedshiftRelationalSpectrumTest {
+ @Test
+ @DisplayName("Glue external tables join internal tables through JDBC and the SDK Data API")
+ void joinsGlueCsvTablesThroughJdbcAndDataApi() throws Exception {
+ String suffix = UUID.randomUUID().toString().replace("-", "").substring(0, 12);
+ String clusterId = "spectrum-rel-" + suffix;
+ String bucket = "spectrum-rel-" + suffix;
+ String database = "spectrum_rel_" + suffix;
+ String roleName = "SpectrumRel" + suffix;
+ boolean clusterCreated = false;
+ boolean bucketCreated = false;
+ boolean databaseCreated = false;
+ boolean roleCreated = false;
+ boolean policyCreated = false;
+ try (RedshiftClient redshift = TestFixtures.redshiftClient();
+ RedshiftDataClient data = TestFixtures.redshiftDataClient();
+ GlueClient glue = TestFixtures.glueClient();
+ S3Client s3 = TestFixtures.s3Client();
+ IamClient iam = TestFixtures.iamClient()) {
+ try {
+ String roleArn = iam.createRole(r -> r.roleName(roleName).assumeRolePolicyDocument("""
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}
+ """)).role().arn();
+ roleCreated = true;
+ iam.putRolePolicy(r -> r.roleName(roleName).policyName("SpectrumRead").policyDocument("""
+ {"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Action":["s3:GetObject","s3:ListBucket","glue:GetTable","glue:GetPartitions"],"Resource":"*"}]}
+ """));
+ policyCreated = true;
+ s3.createBucket(r -> r.bucket(bucket));
+ bucketCreated = true;
+ s3.putObject(r -> r.bucket(bucket).key("events/data.csv"), RequestBody.fromString("1,10\n1,5\n2,7\n"));
+ glue.createDatabase(r -> r.databaseInput(DatabaseInput.builder().name(database).build()));
+ databaseCreated = true;
+ redshift.createCluster(r -> r.clusterIdentifier(clusterId).nodeType("dc2.large")
+ .masterUsername("admin").masterUserPassword("Password123").iamRoles(roleArn));
+ clusterCreated = true;
+ Cluster cluster = redshift.describeClusters(r -> r.clusterIdentifier(clusterId)).clusters().get(0);
+ String url = "jdbc:postgresql://" + cluster.endpoint().address() + ":" + cluster.endpoint().port() + "/dev";
+ try (Connection connection = DriverManager.getConnection(url, "admin", "Password123");
+ Statement setup = connection.createStatement()) {
+ setup.execute("CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE '" + database
+ + "' IAM_ROLE '" + roleArn + "'");
+ setup.execute("CREATE EXTERNAL TABLE lake.events (id INTEGER, amount INTEGER) "
+ + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE LOCATION 's3://"
+ + bucket + "/events/'");
+ setup.execute("CREATE EXTERNAL TABLE lake.mirror_events (id INTEGER, amount INTEGER) "
+ + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE LOCATION 's3://"
+ + bucket + "/events/'");
+ assertEquals("events", glue.getTable(r -> r.databaseName(database).name("events")).table().name());
+ setup.execute("CREATE TABLE customers(id INTEGER, country VARCHAR(2))");
+ setup.execute("INSERT INTO customers VALUES (1,'VN'),(2,'US')");
+ String sql = "SELECT c.country, SUM(e.amount) AS total FROM lake.events e "
+ + "JOIN (SELECT DISTINCT id FROM lake.mirror_events) m ON m.id=e.id "
+ + "JOIN customers c ON c.id=e.id GROUP BY c.country ORDER BY total DESC";
+ try (PreparedStatement query = connection.prepareStatement(sql);
+ ResultSet rows = query.executeQuery()) {
+ assertTrue(rows.next());
+ assertEquals("VN", rows.getString(1));
+ assertEquals(15L, rows.getLong(2));
+ assertTrue(rows.next());
+ assertEquals("US", rows.getString(1));
+ assertEquals(7L, rows.getLong(2));
+ assertFalse(rows.next());
+ }
+ String id = data.executeStatement(r -> r.clusterIdentifier(clusterId).database("dev")
+ .dbUser("admin").sql(sql)).id();
+ Instant deadline = Instant.now().plusSeconds(30);
+ DescribeStatementResponse status = data.describeStatement(r -> r.id(id));
+ while (!status.statusAsString().equals("FINISHED") && Instant.now().isBefore(deadline)) {
+ assertNotEquals("FAILED", status.statusAsString(), status.error());
+ Thread.sleep(100);
+ status = data.describeStatement(r -> r.id(id));
+ }
+ assertEquals("FINISHED", status.statusAsString(), status.error());
+ GetStatementResultResponse result = data.getStatementResult(r -> r.id(id));
+ assertEquals(2L, result.totalNumRows());
+ assertEquals("VN", result.records().get(0).get(0).stringValue());
+ assertEquals(15L, result.records().get(0).get(1).longValue());
+ assertEquals("US", result.records().get(1).get(0).stringValue());
+ String failedBatch = data.batchExecuteStatement(r -> r.clusterIdentifier(clusterId).database("dev")
+ .dbUser("admin").sqls("INSERT INTO customers VALUES (3,'JP')", "SELECT * FROM lake.absent")).id();
+ Instant failureDeadline = Instant.now().plusSeconds(30);
+ DescribeStatementResponse failed = data.describeStatement(r -> r.id(failedBatch));
+ while (!failed.statusAsString().equals("FAILED") && Instant.now().isBefore(failureDeadline)) {
+ assertNotEquals("FINISHED", failed.statusAsString());
+ Thread.sleep(100);
+ failed = data.describeStatement(r -> r.id(failedBatch));
+ }
+ assertEquals("FAILED", failed.statusAsString());
+ try (ResultSet rows = setup.executeQuery("SELECT COUNT(*) FROM customers WHERE id=3")) {
+ assertTrue(rows.next());
+ assertEquals(0L, rows.getLong(1));
+ }
+ }
+ } finally {
+ if (clusterCreated) {
+ redshift.deleteCluster(r -> r.clusterIdentifier(clusterId).skipFinalClusterSnapshot(true));
+ }
+ if (databaseCreated) {
+ glue.deleteDatabase(r -> r.name(database));
+ }
+ if (bucketCreated) {
+ s3.deleteObject(r -> r.bucket(bucket).key("events/data.csv"));
+ s3.deleteBucket(r -> r.bucket(bucket));
+ }
+ if (policyCreated) {
+ iam.deleteRolePolicy(r -> r.roleName(roleName).policyName("SpectrumRead"));
+ }
+ if (roleCreated) {
+ iam.deleteRole(r -> r.roleName(roleName));
+ }
+ }
+ }
+ }
+}
diff --git a/docs/services/redshift-data.md b/docs/services/redshift-data.md
index 49b2d40f86..0c437fb21a 100644
--- a/docs/services/redshift-data.md
+++ b/docs/services/redshift-data.md
@@ -58,6 +58,20 @@ A statement run against a workgroup reports `WorkgroupName` and omits `ClusterId
- **`ExecuteSql` and `BatchExecuteSql`** (the deprecated pre-2020 operations) return `ValidationException`.
- **Type mapping.** JDBC `BOOLEAN` and `BIT` map to `booleanValue`; integer types to `longValue`; floating-point types to `doubleValue`; `NUMERIC` and `DECIMAL` to `stringValue` (as AWS does); binary types to `blobValue`; everything else, including dates, timestamps, and uuid, to `stringValue`. A SQL `NULL` maps to `isNull`. A result column of type `line`, `json`, or `jsonb` fails the statement with the Redshift error text.
+## Spectrum relational queries
+
+The Data API prepares Glue-backed external CSV tables on the same JDBC connection that executes
+the SQL. JOINs with internal tables, multiple external tables, grouping, ordering and parameters
+follow the [Redshift Spectrum relational path](redshift.md#relational-queries-over-glue-backed-external-csv-tables).
+Create the external schema through either the wire endpoint or the Data API; both use the same
+runtime and database binding. Batch preparation runs inside the existing batch transaction, so
+failure rolls back the batch. External read or preparation failures are stored as `FAILED` and
+reported by `DescribeStatement`, as other SQL execution failures are.
+Create external schemas and tables outside a batch: `CREATE EXTERNAL TABLE` cannot run inside a transaction block on AWS, and a batch runs as one transaction, so both statements are rejected there.
+
+This does not add Data API session reuse or asynchronous execution. The legacy Phase 1 catalog
+is not the Glue-backed relational path.
+
## Configuration
| Variable | Default | Description |
diff --git a/docs/services/redshift.md b/docs/services/redshift.md
index b1508e23c3..1a961ba134 100644
--- a/docs/services/redshift.md
+++ b/docs/services/redshift.md
@@ -272,6 +272,63 @@ current S3 emulator authorization mode, and role-specific Spectrum access enforc
future phase. Parquet, JSON, Avro, ORC, partition discovery, `ALTER`, and `DROP` lifecycle
operations are not supported in Phase 1.
+### Relational queries over Glue-backed external CSV tables
+
+An external schema bound to an existing local Glue database supports JOINs between external and
+internal tables, multiple external tables, aliases, bound predicates, GROUP BY, HAVING, ORDER BY
+and LIMIT. PostgreSQL evaluates the original SQL after Floci loads the referenced external
+relations. Simple Query, JDBC Extended Query and the Redshift Data API share this preparation path.
+Extended Query uses native PostgreSQL descriptions and cursors, including fetch-size cursors.
+
+Create the Glue database first, or use `CREATE EXTERNAL DATABASE IF NOT EXISTS`:
+
+```sql
+CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG
+DATABASE 'analytics' IAM_ROLE '<associated-role-arn>'
+CREATE EXTERNAL DATABASE IF NOT EXISTS;
+
+CREATE EXTERNAL TABLE lake.events (customer_id INTEGER, amount INTEGER)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+STORED AS TEXTFILE LOCATION 's3://analytics/events/';
+
+SELECT c.country, SUM(e.amount) AS total
+FROM lake.events e JOIN public.customers c ON e.customer_id = c.id
+GROUP BY c.country ORDER BY total DESC;
+```
+
+This path needs Docker for PostgreSQL and the existing floci-duck runtime used to read external
+data. Use the `floci.duck` configuration described under [Athena](athena.md) to supply a reachable
+runtime when container management is unsuitable. This iteration validates CSV relational reads;
+it does not add new file formats or partition pruning.
+
+Materialized relations are cached in the cluster database. Each new query checks the associated
+role, trust policy and S3 access before using cached data, following the existing S3 enforcement
+configuration. Changes to Glue metadata or S3 objects reload the data. A missing Glue table is an
+error even if its previous contents remain cached. New prepared-statement bindings revalidate
+the data; repeated fetches from one active cursor retain that cursor's result. Loads inside a
+caller transaction do not publish a reusable fingerprint because the transaction may roll back.
+Temporary transfer objects and staging tables are cleaned up independently of cached relations.
+Extended Query loads retain a connection-local fingerprint until its transaction completes;
+the fingerprint becomes shared only after confirmed commit and is discarded on rollback.
+Creating an external schema or table
+inside a transaction, including a Data API batch (which runs as one transaction), is rejected before changing the catalogs: AWS does not allow `CREATE EXTERNAL TABLE` inside a transaction block (SQLSTATE `25001`).
+Cold loads and reloads require database privileges to maintain the materialized table and catalog;
+ordinary users with schema USAGE and table SELECT can read an existing committed cache entry.
+The current floci-duck transfer uses account-scoped development credentials. Cold loads and reloads
+with `FLOCI_SERVICES_S3_ENFORCE_AUTH=true` are not supported by that transfer path yet.
+Role association and trust are still checked before cached reads; S3 policy checks follow the
+configured enforcement mode. This is a Floci limitation, not AWS behavior.
+
+Writes to external relations are rejected. AWS prohibits external UPDATE/DELETE but supports
+some external INSERT operations; external INSERT is an intentional Floci limitation here. Use
+schema-qualified external table names. Full Glue IAM enforcement, Lake Formation, role chaining
+and `IAM_ROLE SESSION` are outside this implementation. Metadata is exposed through
+`svv_external_schemas`, `svv_external_tables`, `svv_external_columns` and `svv_external_partitions`.
+
+Schemas created without a matching Glue database retain the legacy Phase 1 path described above.
+Its single-table query restrictions still apply. PostgreSQL materialization approximates Spectrum
+execution and does not reproduce AWS distributed planning or a cross-service transaction snapshot.
+
### COPY from S3
`COPY <table> [(<columns>)] FROM 's3://<bucket>/<keyOrPrefix>' [options]` sent over the Simple
diff --git a/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckManager.java b/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckManager.java
index db37bfe137..e9a62b5fa8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckManager.java
@@ -122,6 +122,7 @@ public class FlociDuckManager {
.withDynamicPort(DUCK_PORT)
.withDockerNetwork(config.services().dockerNetwork())
.withEmbeddedDns()
+ .withHostDockerInternalOnLinux()
.withLogRotation()
.build();
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
index 798e321d93..869200abd1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java
@@ -603,6 +603,7 @@ public class RedshiftService {
}
containerManager.stop(clusters.accountId(), identifier);
+ proxyManager.forgetSpectrumRuntime(relayKey(clusters.accountId(), identifier));
clusters.delete(identifier);
clusters.flush();
// Invalidate any GetClusterCredentials passwords so a cluster later recreated with this
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java
index 71ec88f443..c89d3903c8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinator.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.redshift.proxy;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayDeque;
import java.util.HashMap;
import java.util.Map;
@@ -106,6 +107,9 @@ final class BackendResponseCoordinator {
void onBackendFrame(char type, byte[] body) {
lock.lock();
try {
+ if (type == 'C') {
+ updateStatusFromCommandTag(new String(body, StandardCharsets.UTF_8));
+ }
if (type == 'Z') {
updateReadyStatus(body);
}
@@ -215,6 +219,28 @@ final class BackendResponseCoordinator {
}
}
+ /**
+ * Tracks transaction boundaries as the tag arrives, so a statement pipelined after COMMIT sees them. The
+ * ROLLBACK tag is also sent for ROLLBACK TO SAVEPOINT, which leaves the transaction open, so only the
+ * ReadyForQuery status can say that a rollback ended it.
+ */
+ private void updateStatusFromCommandTag(String tag) {
+ boolean transactionEnded = switch (tag) {
+ case "BEGIN\0" -> {
+ lastReadyStatus = 'T';
+ yield false;
+ }
+ case "COMMIT\0", "END\0" -> {
+ lastReadyStatus = 'I';
+ yield true;
+ }
+ default -> false;
+ };
+ if (transactionEnded) {
+ session.transactionEnded();
+ }
+ }
+
private void updateReadyStatus(byte[] body) {
if (body != null && body.length == 1) {
lastReadyStatus = (char) body[0];
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java
index ff323c6d1d..57381b0619 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftAuthProxy.java
@@ -5,8 +5,8 @@ import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;
import io.github.hectorvent.floci.services.rds.proxy.PostgresProtocolHandler;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;
import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;
-import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumInterceptor;
+import io.github.hectorvent.floci.services.s3.S3Service;
import org.jboss.logging.Logger;
import java.io.IOException;
@@ -222,7 +222,7 @@ public class RedshiftAuthProxy {
// Redshift-only DDL (DISTKEY/SORTKEY/ENCODE/...) is rewritten for the plain
// PostgreSQL backend on the way through; every other message is relayed verbatim.
new RedshiftInterceptingBridge(session.client(), session.backend(), s3Service, iamService,
- clusterAccountId, iamRoleArns, spectrumInterceptor).run();
+ clusterAccountId, iamRoleArns, spectrumInterceptor, clusterKey, dbName).run();
}
} catch (Exception e) {
LOG.debugv("Redshift connection error for cluster {0}: {1}", clusterKey, e.getMessage());
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
index bfd1085f85..6615d95845 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
@@ -1,10 +1,12 @@
package io.github.hectorvent.floci.services.redshift.proxy;
import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.redshift.spectrum.PostgresExtendedBackendSession;
import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumColumn;
import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumInterceptor;
import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumQueryRewriter;
import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumReadException;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSession;
import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSqlException;
import io.github.hectorvent.floci.services.s3.S3Service;
import org.jboss.logging.Logger;
@@ -60,6 +62,10 @@ public class RedshiftInterceptingBridge {
private final String clusterAccountId;
private final List<String> iamRoleArns;
private final SpectrumInterceptor spectrumInterceptor;
+ private final String clusterKey;
+ private final String databaseName;
+ private final PostgresExtendedBackendSession relationalBackend;
+ private final Map<String, PostgresWireDecoder.ParseMessage> relationalStatements = new HashMap<>();
private final ExtendedQuerySession session = new ExtendedQuerySession();
private final Map<String, SpectrumInterceptor.Plan> spectrumStatements = new HashMap<>();
private final Map<String, SpectrumInterceptor.Plan> spectrumPortals = new HashMap<>();
@@ -70,6 +76,7 @@ public class RedshiftInterceptingBridge {
private volatile boolean pumpBetweenMessages = true;
private volatile boolean pumpFinished = false;
private boolean extendedSpectrumError;
+ private boolean rolledBackInCycle;
public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService) {
this(client, backend, s3Service, iamService, null);
@@ -82,12 +89,13 @@ public class RedshiftInterceptingBridge {
public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService,
String clusterAccountId, List<String> iamRoleArns) {
- this(client, backend, s3Service, iamService, clusterAccountId, iamRoleArns, null);
+ // Without a Spectrum interceptor the cluster key and database name are never read.
+ this(client, backend, s3Service, iamService, clusterAccountId, iamRoleArns, null, null, null);
}
public RedshiftInterceptingBridge(Socket client, Socket backend, S3Service s3Service, IamService iamService,
String clusterAccountId, List<String> iamRoleArns,
- SpectrumInterceptor spectrumInterceptor) {
+ SpectrumInterceptor spectrumInterceptor, String clusterKey, String databaseName) {
this.client = client;
this.backend = backend;
this.s3Service = s3Service;
@@ -95,6 +103,14 @@ public class RedshiftInterceptingBridge {
this.clusterAccountId = clusterAccountId;
this.iamRoleArns = iamRoleArns == null ? List.of() : List.copyOf(iamRoleArns);
this.spectrumInterceptor = spectrumInterceptor;
+ this.clusterKey = clusterKey;
+ this.databaseName = databaseName;
+ this.relationalBackend = new PostgresExtendedBackendSession(backend);
+ }
+
+ private SpectrumSession spectrumSession() {
+ return new SpectrumSession(clusterAccountId, clusterKey, databaseName, iamRoleArns,
+ coordinator.lastReadyStatus() != 'I');
}
@FunctionalInterface
@@ -171,6 +187,9 @@ public class RedshiftInterceptingBridge {
} catch (Exception e) {
LOG.warnv(e, "Unexpected error in RedshiftInterceptingBridge");
} finally {
+ if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null) {
+ spectrumInterceptor.preparation().finishCycle(relationalBackend);
+ }
coordinator.close();
closeQuietly(client, "client");
closeQuietly(backend, "backend");
@@ -185,18 +204,22 @@ public class RedshiftInterceptingBridge {
boolean intercepted;
try {
intercepted = runWithBackendOwned(() -> {
- decision[0] = spectrumInterceptor.intercept(sql, clusterAccountId, "dev", backend);
+ decision[0] = spectrumInterceptor.intercept(sql, spectrumSession(), backend);
return true;
});
- } catch (SpectrumSqlException | SpectrumReadException | IllegalArgumentException exception) {
+ } catch (RuntimeException exception) {
writeSimpleSpectrumError(exception);
return;
}
+ if (!intercepted && spectrumInterceptor.preparation() != null
+ && spectrumInterceptor.preparation().referencesExternal(sql, spectrumSession())) {
+ writeSimpleSpectrumError(new SpectrumReadException("58030", "Unable to prepare external query"));
+ return;
+ }
if (intercepted && decision[0] instanceof SpectrumInterceptor.Decision.Handled) {
coordinator.register(BackendResponseCoordinator.Operation.SIMPLE_QUERY, null);
write(client.getOutputStream(), commandComplete("CREATE EXTERNAL\0"));
- write(client.getOutputStream(), readyForQuery('I'));
- coordinator.onBackendFrame('Z', new byte[]{'I'});
+ reportLocalReady();
return;
}
if (intercepted && decision[0] instanceof SpectrumInterceptor.Decision.Rewritten rewritten) {
@@ -246,16 +269,39 @@ public class RedshiftInterceptingBridge {
// it, per protocol; without this, a later non-Spectrum statement reusing that name would still
// resolve to the stale Spectrum plan in handleBind/handleDescribe/handleExecute.
spectrumStatements.remove(parse.statementName());
+ relationalStatements.remove(parse.statementName());
+ // After a backend error the backend discards everything up to Sync, so a preparation round trip
+ // would only wait out the socket timeout; forward the Parse and let the backend discard it.
+ if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null
+ && !coordinator.isDiscardingUntilSync()
+ && spectrumInterceptor.preparation().referencesExternal(parse.sql(), spectrumSession())) {
+ try {
+ write(backendOut, new byte[]{'H', 0, 0, 0, 4});
+ boolean prepared = runWithBackendOwned(() -> {
+ spectrumInterceptor.preparation().prepare(parse.sql(), spectrumSession(), relationalBackend);
+ return true;
+ });
+ if (!prepared) {
+ throw new SpectrumReadException("58030", "Unable to prepare external query");
+ }
+ relationalStatements.put(parse.statementName(), parse);
+ } catch (RuntimeException exception) {
+ spectrumInterceptor.preparation().finishCycle(relationalBackend);
+ writeParseSpectrumError(exception);
+ return;
+ }
+ }
if (spectrumInterceptor != null && parse.parameterTypeOids().isEmpty()) {
try {
- SpectrumInterceptor.Plan plan = spectrumInterceptor.plan(parse.sql(), clusterAccountId, "dev");
- if (plan instanceof SpectrumInterceptor.Plan.Ddl || plan instanceof SpectrumInterceptor.Plan.Query) {
+ SpectrumInterceptor.Plan plan = spectrumInterceptor.plan(parse.sql(), spectrumSession());
+ if (plan instanceof SpectrumInterceptor.Plan.Ddl || plan instanceof SpectrumInterceptor.Plan.Query
+ || plan instanceof SpectrumInterceptor.Plan.ExternalDdl) {
awaitPriorBackendResponses();
spectrumStatements.put(parse.statementName(), plan);
write(client.getOutputStream(), backendFrame('1', EMPTY_BODY));
return;
}
- } catch (SpectrumSqlException | SpectrumReadException | IllegalArgumentException exception) {
+ } catch (RuntimeException exception) {
writeParseSpectrumError(exception);
return;
}
@@ -285,6 +331,24 @@ public class RedshiftInterceptingBridge {
private void handleBind(PostgresWireDecoder decoder, PostgresWireDecoder.FrontendMessage message,
OutputStream backendOut) throws IOException {
PostgresWireDecoder.BindMessage bind = decoder.decodeBind(message);
+ PostgresWireDecoder.ParseMessage relational = relationalStatements.get(bind.statementName());
+ if (relational != null && !coordinator.isDiscardingUntilSync()) {
+ try {
+ // Parse/Describe responses may be buffered until Flush or Sync; drain them before taking ownership.
+ write(backendOut, new byte[]{'H', 0, 0, 0, 4});
+ boolean prepared = runWithBackendOwned(() -> {
+ spectrumInterceptor.preparation().prepare(relational.sql(), spectrumSession(), relationalBackend);
+ return true;
+ });
+ if (!prepared) {
+ throw new SpectrumReadException("58030", "Unable to prepare external query");
+ }
+ } catch (RuntimeException exception) {
+ spectrumInterceptor.preparation().finishCycle(relationalBackend);
+ writeParseSpectrumError(exception);
+ return;
+ }
+ }
SpectrumInterceptor.Plan spectrumPlan = spectrumStatements.get(bind.statementName());
if (spectrumPlan != null) {
awaitPriorBackendResponses();
@@ -315,7 +379,7 @@ public class RedshiftInterceptingBridge {
}
byte[] response = spectrumDescribeResponse(spectrumPlan);
write(client.getOutputStream(), response);
- } catch (SpectrumSqlException | SpectrumReadException | IllegalArgumentException exception) {
+ } catch (RuntimeException exception) {
writeParseSpectrumError(exception);
}
return;
@@ -425,7 +489,9 @@ public class RedshiftInterceptingBridge {
}
try {
backend.setSoTimeout(EXCHANGE_READ_TIMEOUT_MS);
- ExtendedSpectrumExchange.execute(client, backend, plan, spectrumInterceptor, clusterAccountId, "dev",
+ SpectrumInterceptor.Plan executionPlan = plan instanceof SpectrumInterceptor.Plan.ExternalDdl ddl
+ ? new SpectrumInterceptor.Plan.ExternalDdl(ddl.sql(), spectrumSession()) : plan;
+ ExtendedSpectrumExchange.execute(client, backend, executionPlan, spectrumInterceptor, clusterAccountId, databaseName,
coordinator, ticket, portalName, maxRows, spectrumCursors);
} finally {
try {
@@ -442,6 +508,7 @@ public class RedshiftInterceptingBridge {
PostgresWireDecoder.TargetMessage close = decoder.decodeClose(message);
boolean spectrumOwned;
if (close.targetType() == 'S') {
+ relationalStatements.remove(close.name());
SpectrumInterceptor.Plan closedPlan = spectrumStatements.remove(close.name());
spectrumOwned = closedPlan != null;
if (closedPlan != null) {
@@ -602,8 +669,14 @@ public class RedshiftInterceptingBridge {
private void writeSimpleSpectrumError(RuntimeException exception) throws IOException {
coordinator.register(BackendResponseCoordinator.Operation.SIMPLE_QUERY, null);
write(client.getOutputStream(), errorResponse(spectrumSqlState(exception), exception.getMessage()));
- write(client.getOutputStream(), readyForQuery('I'));
- coordinator.onBackendFrame('Z', new byte[]{'I'});
+ reportLocalReady();
+ }
+
+ /** A query answered locally never reaches the backend, so its transaction state must be reported unchanged. */
+ private void reportLocalReady() throws IOException {
+ char status = coordinator.lastReadyStatus();
+ write(client.getOutputStream(), readyForQuery(status));
+ coordinator.onBackendFrame('Z', new byte[]{(byte) status});
}
private void writeParseSpectrumError(RuntimeException exception) throws IOException {
@@ -643,7 +716,11 @@ public class RedshiftInterceptingBridge {
if (exception instanceof SpectrumReadException spectrumReadException) {
return spectrumReadException.sqlState();
}
- return "22023";
+ if (exception instanceof IllegalArgumentException) {
+ return "22023";
+ }
+ LOG.warnv(exception, "External query preparation failed unexpectedly");
+ return "58030";
}
/**
@@ -710,7 +787,38 @@ public class RedshiftInterceptingBridge {
}
private void pumpBackendToClient() {
- WireFrameTracker tracker = new WireFrameTracker(coordinator::onBackendFrame);
+ WireFrameTracker tracker = new WireFrameTracker((type, body) -> {
+ // Publish or drop pending preparation state before the coordinator reports the backend idle,
+ // so the next preparation already sees a schema bound by a committed CREATE EXTERNAL SCHEMA.
+ if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null) {
+ if (type == 'E') {
+ spectrumInterceptor.preparation().finishCycle(relationalBackend);
+ }
+ if (type == 'C' && "ROLLBACK\0".equals(new String(body, StandardCharsets.UTF_8))) {
+ // Loads are dropped on every rollback; bindings and Glue tables only once ReadyForQuery
+ // reports the transaction closed, because ROLLBACK TO SAVEPOINT keeps it open.
+ rolledBackInCycle = true;
+ spectrumInterceptor.preparation().discardPendingLoads(relationalBackend);
+ }
+ if (type == 'Z' && body.length > 0 && body[0] == 'I') {
+ if (rolledBackInCycle) {
+ spectrumInterceptor.preparation().finishCycle(relationalBackend);
+ } else {
+ spectrumInterceptor.preparation().finishCycle(relationalBackend, true);
+ }
+ }
+ if (type == 'Z') {
+ rolledBackInCycle = false;
+ }
+ }
+ if (type == 'E') {
+ relationalBackend.onBackendError();
+ }
+ coordinator.onBackendFrame(type, body);
+ if (type == 'Z') {
+ relationalBackend.onSync();
+ }
+ });
try {
InputStream backendIn = backend.getInputStream();
OutputStream clientOut = client.getOutputStream();
@@ -770,6 +878,7 @@ public class RedshiftInterceptingBridge {
private long declaredBodyLength = 0;
private char pendingType = 0;
private byte readyStatus;
+ private byte[] commandTag = EMPTY_BODY;
WireFrameTracker(FrameCompletion onFrame) {
this.onFrame = onFrame;
@@ -786,6 +895,9 @@ public class RedshiftInterceptingBridge {
if (pendingType == 'Z' && declaredBodyLength == 1) {
readyStatus = (byte) current;
}
+ if (pendingType == 'C' && commandTag.length > 0) {
+ commandTag[(int) (declaredBodyLength - bodyRemaining)] = (byte) current;
+ }
bodyRemaining--;
if (bodyRemaining == 0) {
publishCompletion();
@@ -799,6 +911,7 @@ public class RedshiftInterceptingBridge {
pendingType = (char) (header[0] & 0xFF);
bodyRemaining = Math.max(0, msgLen - 4);
declaredBodyLength = bodyRemaining;
+ commandTag = pendingType == 'C' && bodyRemaining <= 128 ? new byte[(int) bodyRemaining] : EMPTY_BODY;
headerBytesSeen = 0;
if (bodyRemaining == 0) {
publishCompletion();
@@ -809,7 +922,7 @@ public class RedshiftInterceptingBridge {
private void publishCompletion() {
byte[] body = pendingType == 'Z' && declaredBodyLength == 1
- ? new byte[]{readyStatus} : new byte[0];
+ ? new byte[]{readyStatus} : pendingType == 'C' ? commandTag : EMPTY_BODY;
onFrame.accept(pendingType, body);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java
index 0fafb683d5..cd09cb0a38 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftProxyManager.java
@@ -5,8 +5,8 @@ import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.rds.proxy.PasswordValidator;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyTlsCertificates;
import io.github.hectorvent.floci.services.rds.proxy.RdsSigV4Validator;
-import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumInterceptor;
+import io.github.hectorvent.floci.services.s3.S3Service;
import io.quarkus.runtime.ShutdownEvent;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Observes;
@@ -76,6 +76,7 @@ public class RedshiftProxyManager {
// Make sure the self-signed proxy certificate covers the host clients will connect to,
// so sslmode=prefer/require handshakes succeed.
tlsCertificates.ensureHost(advertisedHost);
+ invalidateSpectrumRuntime(relayKey);
EmulatorConfig.RedshiftServiceConfig redshiftConfig = config.services().redshift();
String clusterAccountId = relayKey.substring(0, relayKey.indexOf(':'));
RedshiftAuthProxy proxy = new RedshiftAuthProxy(
@@ -138,6 +139,20 @@ public class RedshiftProxyManager {
proxies.remove(relayKey);
LOG.infov("Stopped Redshift proxy for cluster {0}", relayKey);
}
+ invalidateSpectrumRuntime(relayKey);
+ }
+
+ public void forgetSpectrumRuntime(String relayKey) {
+ if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null) {
+ String accountId = relayKey.substring(0, relayKey.indexOf(':'));
+ spectrumInterceptor.preparation().forgetRuntime(accountId, relayKey);
+ }
+ }
+
+ private void invalidateSpectrumRuntime(String relayKey) {
+ if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null) {
+ spectrumInterceptor.preparation().invalidateRuntime(relayKey);
+ }
}
public synchronized void stopAll() {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java
index 11e541fc26..91b5f79c42 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java
@@ -3,6 +3,10 @@ package io.github.hectorvent.floci.services.redshift.spectrum;
import java.io.InputStream;
public interface BackendSql {
+ default boolean permitsCachePublication() {
+ return true;
+ }
+
void execute(String sql);
long copyIn(String copySql, InputStream data);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java
index 66d6c10f63..39eddc4569 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java
@@ -68,6 +68,32 @@ public class ExternalTableMaterializer {
private final ConcurrentHashMap<String, String> schemaSignatures = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, ReentrantLock> locks = new ConcurrentHashMap<>();
private final Set<String> sweptAccounts = ConcurrentHashMap.newKeySet();
+ private final Map<BackendSql, Map<String, PendingLoad>> pendingLoads = new ConcurrentHashMap<>();
+ private final Map<String, PendingLoad> latestLoads = new ConcurrentHashMap<>();
+
+ private record PendingLoad(String fingerprint, String definitions) { }
+
+ public void finishCycle(BackendSql backend) {
+ pendingLoads.remove(backend);
+ }
+
+ public void finishCycle(BackendSql backend, boolean committed) {
+ Map<String, PendingLoad> completed = pendingLoads.remove(backend);
+ if (committed && completed != null) {
+ completed.forEach((key, loaded) -> {
+ ReentrantLock lock = locks.computeIfAbsent(key, ignored -> new ReentrantLock());
+ lock.lock();
+ try {
+ if (latestLoads.get(key) == loaded) {
+ fingerprints.put(key, loaded.fingerprint());
+ schemaSignatures.put(key, loaded.definitions());
+ }
+ } finally {
+ lock.unlock();
+ }
+ });
+ }
+ }
public ExternalTableMaterializer(FlociDuckClient duckClient, GlueService glueService, S3Service s3Service,
IamService iamService, EmulatorConfig config) {
@@ -107,6 +133,12 @@ public class ExternalTableMaterializer {
// An Iceberg snapshot can change through its metadata or manifests while the Glue table and the
// listed objects stay the same, so no fingerprint can vouch for it: it is read on every query.
boolean cacheable = !session.inTransaction() && !GlueTableResolver.isIcebergTable(table);
+ Map<String, PendingLoad> localLoads = backend.permitsCachePublication() ? Map.of()
+ : pendingLoads.computeIfAbsent(backend, ignored -> new ConcurrentHashMap<>());
+ PendingLoad local = localLoads.get(cacheKey);
+ if (!GlueTableResolver.isIcebergTable(table) && local != null && fingerprint.equals(local.fingerprint())) {
+ return Outcome.CURRENT;
+ }
if (cacheable && fingerprint.equals(fingerprints.get(cacheKey))) {
return Outcome.CURRENT;
}
@@ -116,9 +148,18 @@ public class ExternalTableMaterializer {
if (cacheable && fingerprint.equals(fingerprints.get(cacheKey))) {
return Outcome.CURRENT;
}
- String previousSignature = schemaSignatures.get(cacheKey);
+ String previousSignature = local == null ? schemaSignatures.get(cacheKey) : local.definitions();
+ latestLoads.remove(cacheKey);
String definitions = load(backend, session, binding, table, sources, previousSignature);
- if (session.inTransaction()) {
+ if (!backend.permitsCachePublication()) {
+ fingerprints.remove(cacheKey);
+ PendingLoad loaded = new PendingLoad(fingerprint, definitions);
+ localLoads.put(cacheKey, loaded);
+ latestLoads.put(cacheKey, loaded);
+ if (!definitions.equals(schemaSignatures.get(cacheKey))) {
+ schemaSignatures.remove(cacheKey);
+ }
+ } else if (session.inTransaction()) {
// Transactional DDL may roll back, so the rows cannot be trusted afterwards. The schema
// signature stays when this load kept the committed columns, so a view that depends on the
// table can still be refilled in place; a replaced table may roll back to the old columns.
@@ -182,6 +223,7 @@ public class ExternalTableMaterializer {
private void forgetMatching(Predicate<String> matches) {
fingerprints.keySet().removeIf(matches);
schemaSignatures.keySet().removeIf(matches);
+ latestLoads.keySet().removeIf(matches);
// Keep locks: removing a held lock lets another thread create a second lock for the same table.
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresExtendedBackendSession.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresExtendedBackendSession.java
new file mode 100644
index 0000000000..365ce179cb
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresExtendedBackendSession.java
@@ -0,0 +1,203 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.util.UUID;
+
+/** Runs preparation without ending the client's transaction or destroying its portals. */
+public final class PostgresExtendedBackendSession implements BackendSql {
+ private final Socket backend;
+ private String name;
+ private volatile boolean failedUntilSync;
+
+ public PostgresExtendedBackendSession(Socket backend) {
+ this.backend = backend;
+ }
+
+ @Override
+ public boolean permitsCachePublication() {
+ return false;
+ }
+
+ /** The client's own pipelined statement failed; the backend discards everything until Sync. */
+ public void onBackendError() {
+ failedUntilSync = true;
+ }
+
+ public void onSync() {
+ failedUntilSync = false;
+ }
+
+ @Override
+ public void execute(String sql) {
+ try {
+ start(atomicCommand(sql));
+ closeAndFlush();
+ awaitClosed();
+ } catch (IOException exception) {
+ throw failure(exception);
+ }
+ }
+
+ public static String atomicCommand(String sql) {
+ if (!sql.startsWith("CREATE ") && !sql.startsWith("DROP ")) {
+ return sql;
+ }
+ String delimiter = "$floci_" + UUID.randomUUID().toString().replace("-", "") + "$";
+ return "DO " + delimiter + " BEGIN " + sql + "; END " + delimiter;
+ }
+
+ @Override
+ public long copyIn(String sql, InputStream data) {
+ try {
+ start(sql);
+ frame('H', new byte[0]);
+ backend.getOutputStream().flush();
+ readUntil('G');
+ byte[] chunk = new byte[8192];
+ try {
+ int count;
+ while ((count = data.read(chunk)) >= 0) {
+ if (count > 0) {
+ frame('d', chunk, count);
+ }
+ }
+ } catch (IOException exception) {
+ frame('f', "Unable to read external data\0".getBytes(StandardCharsets.UTF_8));
+ backend.getOutputStream().flush();
+ try {
+ readUntil('C');
+ } catch (SpectrumReadException expected) {
+ // COPY failure leaves recovery to the client's Sync.
+ exception.addSuppressed(expected);
+ }
+ throw exception;
+ }
+ frame('c', new byte[0]);
+ closeAndFlush();
+ String tag = awaitClosed();
+ return Long.parseLong(tag.substring(tag.lastIndexOf(' ') + 1).trim());
+ } catch (IOException exception) {
+ throw failure(exception);
+ }
+ }
+
+ private void start(String sql) throws IOException {
+ if (failedUntilSync) {
+ throw new SpectrumReadException("25P02", "Preparation is waiting for Sync after a backend error");
+ }
+ name = "floci_prepare_" + UUID.randomUUID().toString().replace("-", "");
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ DataOutputStream body = new DataOutputStream(bytes);
+ cstring(body, name);
+ cstring(body, sql);
+ body.writeShort(0);
+ frame('P', bytes.toByteArray());
+ bytes.reset();
+ cstring(body, name);
+ cstring(body, name);
+ body.writeShort(0);
+ body.writeShort(0);
+ body.writeShort(0);
+ frame('B', bytes.toByteArray());
+ bytes.reset();
+ cstring(body, name);
+ body.writeInt(0);
+ frame('E', bytes.toByteArray());
+ }
+
+ private void closeAndFlush() throws IOException {
+ frame('C', ("P" + name + "\0").getBytes(StandardCharsets.UTF_8));
+ frame('C', ("S" + name + "\0").getBytes(StandardCharsets.UTF_8));
+ frame('H', new byte[0]);
+ backend.getOutputStream().flush();
+ }
+
+ private String awaitClosed() throws IOException {
+ String tag = "";
+ for (int closed = 0; closed < 2;) {
+ Response response = read();
+ if (response.type() == '3') {
+ closed++;
+ } else if (response.type() == 'C') {
+ tag = new String(response.body(), StandardCharsets.UTF_8).replace("\0", "");
+ }
+ }
+ return tag;
+ }
+
+ private void readUntil(char type) throws IOException {
+ Response response;
+ do {
+ // ParseComplete, BindComplete and asynchronous messages precede the response.
+ response = read();
+ if (response.type() == 'Z') {
+ throw new IOException("Backend ended preparation before sending " + type);
+ }
+ } while (response.type() != type);
+ }
+
+ private Response read() throws IOException {
+ DataInputStream input = new DataInputStream(backend.getInputStream());
+ char type = (char) input.readUnsignedByte();
+ int length = input.readInt();
+ if (length < 4 || length > 64 * 1024 * 1024) {
+ throw new IOException("Invalid preparation response length");
+ }
+ byte[] body = input.readNBytes(length - 4);
+ if (body.length != length - 4) {
+ throw new IOException("Backend closed during preparation");
+ }
+ if (type == 'E') {
+ failedUntilSync = true;
+ String state = "58030";
+ String message = "External query preparation failed";
+ for (int offset = 0; offset < body.length && body[offset] != 0;) {
+ char field = (char) body[offset++];
+ int start = offset;
+ while (offset < body.length && body[offset] != 0) {
+ offset++;
+ }
+ String value = new String(body, start, offset - start, StandardCharsets.UTF_8);
+ if (field == 'C') {
+ state = value;
+ } else if (field == 'M') {
+ message = value;
+ }
+ offset++;
+ }
+ throw new SpectrumReadException(state, message);
+ }
+ return new Response(type, body);
+ }
+
+ private void frame(char type, byte[] body) throws IOException {
+ frame(type, body, body.length);
+ }
+
+ /** Writes the frame in one call; a split write would put a header packet on the wire of its own. */
+ private void frame(char type, byte[] body, int length) throws IOException {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream(length + 5);
+ DataOutputStream output = new DataOutputStream(bytes);
+ output.writeByte(type);
+ output.writeInt(length + 4);
+ output.write(body, 0, length);
+ backend.getOutputStream().write(bytes.toByteArray());
+ }
+
+ private static void cstring(DataOutputStream output, String text) throws IOException {
+ output.write(text.getBytes(StandardCharsets.UTF_8));
+ output.writeByte(0);
+ }
+
+ private static SpectrumReadException failure(IOException exception) {
+ return new SpectrumReadException("58030", "Unable to prepare external query", exception);
+ }
+
+ private record Response(char type, byte[] body) { }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java
index 13820d901b..afa68e92ed 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.redshift.spectrum;
import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
import java.net.Socket;
import java.util.Optional;
@@ -14,16 +15,48 @@ public final class SpectrumInterceptor {
private final SpectrumQueryRewriter queryRewriter;
private final SpectrumS3Reader reader;
private final SpectrumMaterializer materializer;
+ private final SpectrumQueryPreparation preparation;
public SpectrumInterceptor(SpectrumCatalog catalog, SpectrumStatementParser statementParser,
SpectrumQueryClassifier queryClassifier, SpectrumQueryRewriter queryRewriter,
SpectrumS3Reader reader, SpectrumMaterializer materializer) {
+ this(catalog, statementParser, queryClassifier, queryRewriter, reader, materializer, null);
+ }
+
+ @Inject
+ public SpectrumInterceptor(SpectrumCatalog catalog, SpectrumStatementParser statementParser,
+ SpectrumQueryClassifier queryClassifier, SpectrumQueryRewriter queryRewriter,
+ SpectrumS3Reader reader, SpectrumMaterializer materializer,
+ SpectrumQueryPreparation preparation) {
this.catalog = catalog;
this.statementParser = statementParser;
this.queryClassifier = queryClassifier;
this.queryRewriter = queryRewriter;
this.reader = reader;
this.materializer = materializer;
+ this.preparation = preparation;
+ }
+
+ public SpectrumQueryPreparation preparation() {
+ return preparation;
+ }
+
+ public Plan plan(String sql, SpectrumSession session) {
+ if (preparation != null && preparation.handlesDdl(sql, session)) {
+ return new Plan.ExternalDdl(sql, session);
+ }
+ if (preparation != null && preparation.referencesExternal(sql, session)) {
+ return new Plan.Forward();
+ }
+ return plan(sql, session.accountId(), session.databaseName());
+ }
+
+ public Decision intercept(String sql, SpectrumSession session, Socket backend) {
+ Plan plan = plan(sql, session);
+ if (plan instanceof Plan.Forward && preparation != null) {
+ preparation.prepare(sql, session, new PostgresBackendSession(backend));
+ }
+ return execute(plan, session.accountId(), session.databaseName(), backend);
}
public Decision intercept(String sql, String accountId, String databaseName, Socket backend) {
@@ -54,6 +87,12 @@ public final class SpectrumInterceptor {
public Decision execute(Plan plan, String accountId, String databaseName, Socket backend) {
switch (plan) {
+ case Plan.ExternalDdl(String sql, SpectrumSession session) -> {
+ if (preparation.prepare(sql, session, new PostgresBackendSession(backend))) {
+ return new Decision.Handled();
+ }
+ return execute(plan(sql, accountId, databaseName), accountId, databaseName, backend);
+ }
case Plan.Ddl(SpectrumStatement parsed) -> {
switch (parsed) {
case SpectrumStatement.CreateSchema schema -> catalog.createSchema(new SpectrumExternalSchema(
@@ -91,7 +130,9 @@ public final class SpectrumInterceptor {
materializer.cleanup(backend, materialization);
}
- public sealed interface Plan permits Plan.Ddl, Plan.Forward, Plan.Query {
+ public sealed interface Plan permits Plan.Ddl, Plan.Forward, Plan.Query, Plan.ExternalDdl {
+ record ExternalDdl(String sql, SpectrumSession session) implements Plan {
+ }
record Ddl(SpectrumStatement statement) implements Plan {
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparation.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparation.java
new file mode 100644
index 0000000000..8a4c3a0d3b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparation.java
@@ -0,0 +1,195 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Database;
+import jakarta.enterprise.context.ApplicationScoped;
+import org.jboss.logging.Logger;
+
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+@ApplicationScoped
+public class SpectrumQueryPreparation {
+ private static final Logger LOG = Logger.getLogger(SpectrumQueryPreparation.class);
+ private final ExternalCatalogRegistry registry;
+ private final ExternalTableMaterializer materializer;
+ private final ExternalStatementParser parser;
+ private final GlueService glue;
+ private final ExternalMetadataWriter metadata;
+ private final Map<BackendSql, List<ExternalSchemaBinding>> pendingBindings = new ConcurrentHashMap<>();
+
+ public SpectrumQueryPreparation(ExternalCatalogRegistry registry, ExternalTableMaterializer materializer,
+ ExternalStatementParser parser, GlueService glue, ExternalMetadataWriter metadata) {
+ this.registry = registry;
+ this.materializer = materializer;
+ this.parser = parser;
+ this.glue = glue;
+ this.metadata = metadata;
+ }
+
+ public boolean prepare(String sql, SpectrumSession session, BackendSql backend) {
+ try {
+ return RequestScopes.callAs(session.accountId(), () -> prepareInScope(sql, session, backend));
+ } catch (AwsException exception) {
+ String state = switch (exception.getErrorCode()) {
+ case "AlreadyExistsException" -> "42P07";
+ case "EntityNotFoundException" -> "42P01";
+ case "AccessDenied", "AccessDeniedException" -> "42501";
+ case "InvalidInputException" -> "42601";
+ default -> "58030";
+ };
+ throw new SpectrumSqlException(state, exception.getMessage());
+ }
+ }
+
+ public boolean referencesExternal(String sql, SpectrumSession session) {
+ Set<String> schemas = bindings(session).keySet();
+ return !ExternalReferenceScanner.scan(sql, schemas).isEmpty()
+ || ExternalReferenceScanner.writeTarget(sql, schemas).isPresent();
+ }
+
+ public boolean handlesDdl(String sql, SpectrumSession session) {
+ Optional<ExternalStatement> statement = parser.parse(sql);
+ return statement.orElse(null) instanceof ExternalStatement.CreateSchema
+ || statement.orElse(null) instanceof ExternalStatement.CreateTable table
+ && (session.inTransaction() || bindings(session).containsKey(table.schemaName()));
+ }
+
+ private boolean prepareInScope(String sql, SpectrumSession session, BackendSql backend) {
+ Optional<ExternalStatement> statement = parser.parse(sql);
+ if (statement.orElse(null) instanceof ExternalStatement.CreateSchema schema) {
+ return createSchema(schema, session, backend);
+ }
+ Map<String, ExternalSchemaBinding> bindings = bindings(session);
+ if (statement.orElse(null) instanceof ExternalStatement.CreateTable && session.inTransaction()) {
+ // Legacy-catalog tables are saved immediately too, so a rollback would leave them behind.
+ throw new SpectrumSqlException("25001", "CREATE EXTERNAL TABLE cannot run inside a transaction block");
+ }
+ if (statement.orElse(null) instanceof ExternalStatement.CreateTable table && bindings.containsKey(table.schemaName())) {
+ ExternalSchemaBinding binding = bindings.get(table.schemaName());
+ glue.createTable(binding.glueDatabase(), GlueTableBuilder.toGlueTable(table));
+ metadata.refresh(backend, session.accountId(), binding);
+ return true;
+ }
+ Optional<ExternalReferenceScanner.Reference> write = ExternalReferenceScanner.writeTarget(sql, bindings.keySet());
+ if (write.isPresent()) {
+ throw new SpectrumSqlException("0A000", "Writes to external relation \""
+ + write.get().schema() + "." + write.get().table() + "\" are not supported");
+ }
+ Set<String> refreshed = new HashSet<>();
+ for (ExternalReferenceScanner.Reference reference : ExternalReferenceScanner.scan(sql, bindings.keySet())) {
+ ExternalSchemaBinding binding = bindings.get(reference.schema());
+ ExternalTableMaterializer.Outcome outcome = materializer.ensureCurrent(backend, session, binding, reference.table());
+ if (outcome == ExternalTableMaterializer.Outcome.NOT_EXTERNAL) {
+ throw new SpectrumSqlException("42P01", "external relation \"" + reference.schema()
+ + "." + reference.table() + "\" does not exist");
+ }
+ if (outcome == ExternalTableMaterializer.Outcome.LOADED && refreshed.add(binding.schemaName())) {
+ metadata.refresh(backend, session.accountId(), binding);
+ }
+ }
+ return false;
+ }
+
+ private boolean createSchema(ExternalStatement.CreateSchema schema, SpectrumSession session, BackendSql backend) {
+ if (session.inTransaction()) {
+ throw new SpectrumSqlException("0A000", "CREATE EXTERNAL SCHEMA inside a transaction is not supported");
+ }
+ Optional<ExternalSchemaBinding> existing = registry.find(session.accountId(), session.clusterKey(),
+ session.databaseName(), schema.schemaName());
+ if (existing.isPresent()) {
+ if (schema.ifNotExists()) {
+ return true;
+ }
+ throw new SpectrumSqlException("42P06", "schema \"" + schema.schemaName() + "\" already exists");
+ }
+ try {
+ glue.getDatabase(schema.glueDatabase());
+ } catch (AwsException exception) {
+ if (!"EntityNotFoundException".equals(exception.getErrorCode())) {
+ throw exception;
+ }
+ if (!schema.createDatabaseIfNotExists()) {
+ // Preserve the legacy catalog path for schemas without a Glue database.
+ return false;
+ }
+ Database database = new Database();
+ database.setName(schema.glueDatabase());
+ glue.createDatabase(database);
+ }
+ ExternalSchemaBinding binding = new ExternalSchemaBinding(session.accountId(), session.clusterKey(),
+ session.databaseName(), schema.schemaName(), schema.glueDatabase(), schema.iamRoleArn());
+ backend.execute("CREATE SCHEMA " + ExternalTableMaterializer.quote(schema.schemaName()));
+ try {
+ metadata.refresh(backend, session.accountId(), binding);
+ bindWhenCommitted(backend, binding);
+ } catch (RuntimeException exception) {
+ try {
+ backend.execute("DROP SCHEMA " + ExternalTableMaterializer.quote(schema.schemaName()) + " CASCADE");
+ } catch (RuntimeException cleanupFailure) {
+ LOG.warnv(cleanupFailure, "Could not clean up failed external schema {0}", schema.schemaName());
+ }
+ throw exception;
+ }
+ return true;
+ }
+
+ private Map<String, ExternalSchemaBinding> bindings(SpectrumSession session) {
+ Map<String, ExternalSchemaBinding> bindings = new LinkedHashMap<>();
+ for (ExternalSchemaBinding binding : registry.list(session.accountId(), session.clusterKey(), session.databaseName())) {
+ bindings.put(binding.schemaName(), binding);
+ }
+ return bindings;
+ }
+
+ public void forgetRuntime(String accountId, String clusterKey) {
+ registry.removeCluster(accountId, clusterKey);
+ materializer.forgetCluster(clusterKey);
+ }
+
+ public void invalidateRuntime(String clusterKey) {
+ materializer.forgetCluster(clusterKey);
+ }
+
+ /**
+ * A backend that shares the client's transaction can still roll the schema back, so the binding is held
+ * until {@link #finishCycle(BackendSql, boolean)} sees the commit. Any other backend has committed already.
+ */
+ private void bindWhenCommitted(BackendSql backend, ExternalSchemaBinding binding) {
+ if (backend.permitsCachePublication()) {
+ registry.bind(binding);
+ return;
+ }
+ pendingBindings.computeIfAbsent(backend, ignored -> new CopyOnWriteArrayList<>()).add(binding);
+ }
+
+ /** The transaction rolled back: drop everything staged for it. */
+ public void finishCycle(BackendSql backend) {
+ pendingBindings.remove(backend);
+ materializer.finishCycle(backend);
+ }
+
+ public void finishCycle(BackendSql backend, boolean committed) {
+ List<ExternalSchemaBinding> completed = pendingBindings.remove(backend);
+ if (committed && completed != null) {
+ completed.forEach(registry::bind);
+ }
+ materializer.finishCycle(backend, committed);
+ }
+
+ /**
+ * A rollback to a savepoint leaves the transaction open, so only the loads are dropped: they may have been
+ * made after the savepoint, and their fingerprints would no longer match the backend tables.
+ */
+ public void discardPendingLoads(BackendSql backend) {
+ materializer.finishCycle(backend);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSql.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSql.java
new file mode 100644
index 0000000000..5b697b3bf4
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSql.java
@@ -0,0 +1,44 @@
+package io.github.hectorvent.floci.services.redshiftdata;
+
+import io.github.hectorvent.floci.services.redshift.spectrum.BackendSql;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumReadException;
+import org.postgresql.PGConnection;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+final class RedshiftDataBackendSql implements BackendSql {
+ private final Connection connection;
+
+ RedshiftDataBackendSql(Connection connection) {
+ this.connection = connection;
+ }
+
+ @Override
+ public void execute(String sql) {
+ try (Statement statement = connection.createStatement()) {
+ statement.execute(sql);
+ } catch (SQLException exception) {
+ throw failure(exception);
+ }
+ }
+
+ @Override
+ public long copyIn(String copySql, InputStream data) {
+ try {
+ return connection.unwrap(PGConnection.class).getCopyAPI().copyIn(copySql, data);
+ } catch (SQLException exception) {
+ throw failure(exception);
+ } catch (IOException exception) {
+ throw new SpectrumReadException("58030", "Unable to stream external table data", exception);
+ }
+ }
+
+ private static SpectrumReadException failure(SQLException exception) {
+ String state = exception.getSQLState() == null ? "58030" : exception.getSQLState();
+ return new SpectrumReadException(state, exception.getMessage(), exception);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java
index e2029564d5..0ef79e221b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolver.java
@@ -8,6 +8,8 @@ import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.services.redshift.RedshiftCredentialBroker;
import io.github.hectorvent.floci.services.redshift.RedshiftService;
import io.github.hectorvent.floci.services.redshift.model.Cluster;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSession;
+import io.github.hectorvent.floci.services.redshiftserverless.RedshiftServerlessRuntime;
import io.github.hectorvent.floci.services.redshiftserverless.RedshiftServerlessService;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;
@@ -65,17 +67,21 @@ class RedshiftDataResourceResolver {
}
RedshiftServerlessService.WorkgroupTarget workgroup =
serverlessService.getWorkgroupTarget(request.get("WorkgroupName").asText(), region);
+ String accountId = AwsArnUtils.parse(workgroup.arn()).accountId();
+ SpectrumSession spectrum = new SpectrumSession(accountId,
+ accountId + ":" + RedshiftServerlessRuntime.backendId(region, workgroup.workgroupName()), database,
+ workgroup.iamRoleArns(), false);
if (!database.equals(workgroup.database())) {
throw validation("Database " + database + " does not exist in workgroup " + workgroup.workgroupName()
+ "; the only database is " + workgroup.database() + ".");
}
if (!hasText(request, "SecretArn")) {
return new DatabaseTarget(workgroup.arn(), workgroup.host(), workgroup.port(), database,
- workgroup.masterUsername(), workgroup.masterPassword());
+ workgroup.masterUsername(), workgroup.masterPassword(), spectrum);
}
Credentials creds = secretCredentials(request.get("SecretArn").asText(), region);
return new DatabaseTarget(workgroup.arn(), workgroup.host(), workgroup.port(), database,
- creds.username(), creds.password());
+ creds.username(), creds.password(), spectrum);
}
private Credentials secretCredentials(String secretArn, String region) {
@@ -146,7 +152,10 @@ class RedshiftDataResourceResolver {
if (host == null || host.isBlank() || port <= 0) {
throw validation("Cluster runtime is not available for Data API execution.");
}
- return new DatabaseTarget(arn, host, port, database, user, password);
+ String accountId = AwsArnUtils.parse(arn).accountId();
+ SpectrumSession spectrum = new SpectrumSession(accountId, accountId + ":" + cluster.getClusterIdentifier(),
+ database, cluster.getIamRoleArns(), false);
+ return new DatabaseTarget(arn, host, port, database, user, password, spectrum);
}
private Credentials parseCredentials(String secretString) {
@@ -196,7 +205,11 @@ class RedshiftDataResourceResolver {
return value == null || value.isNull() ? null : value.asText();
}
- record DatabaseTarget(String arn, String host, int port, String database, String user, String password) {
+ record DatabaseTarget(String arn, String host, int port, String database, String user, String password,
+ SpectrumSession spectrum) {
+ DatabaseTarget(String arn, String host, int port, String database, String user, String password) {
+ this(arn, host, port, database, user, password, null);
+ }
}
private record Credentials(String username, String password) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java
index 5c0382d39a..2aa84383d0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java
@@ -7,6 +7,11 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.common.SqlParameterParser.ParsedSql;
+import io.github.hectorvent.floci.services.redshift.spectrum.BackendSql;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumQueryPreparation;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumReadException;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSession;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSqlException;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -33,16 +38,25 @@ public class RedshiftDataService implements Resettable {
private final RedshiftDataConnectionFactory connectionFactory;
private final RedshiftDataStatementStore store;
private final ObjectMapper objectMapper;
+ private final SpectrumQueryPreparation preparation;
- @Inject
public RedshiftDataService(RedshiftDataResourceResolver resolver,
RedshiftDataConnectionFactory connectionFactory,
RedshiftDataStatementStore store,
ObjectMapper objectMapper) {
+ this(resolver, connectionFactory, store, objectMapper, null);
+ }
+
+ @Inject
+ public RedshiftDataService(RedshiftDataResourceResolver resolver,
+ RedshiftDataConnectionFactory connectionFactory,
+ RedshiftDataStatementStore store, ObjectMapper objectMapper,
+ SpectrumQueryPreparation preparation) {
this.resolver = resolver;
this.connectionFactory = connectionFactory;
this.store = store;
this.objectMapper = objectMapper;
+ this.preparation = preparation;
}
@Override
@@ -77,7 +91,7 @@ public class RedshiftDataService implements Resettable {
RedshiftDataResourceResolver.DatabaseTarget target,
String sql, Map<String, String> parameters) {
try (Connection connection = connectionFactory.open(target)) {
- runOnConnection(stored, connection, sql, parameters);
+ runOnConnection(stored, connection, sql, parameters, target.spectrum());
} catch (SQLException e) {
stored.status = RedshiftDataStatementStore.Status.FAILED;
stored.error = e.getMessage();
@@ -104,8 +118,25 @@ public class RedshiftDataService implements Resettable {
}
private void runOnConnection(RedshiftDataStatementStore.StoredStatement stored,
- Connection connection, String sql, Map<String, String> parameters)
+ Connection connection, String sql, Map<String, String> parameters, SpectrumSession context)
throws SQLException {
+ if (preparation != null && context != null) {
+ SpectrumSession session = new SpectrumSession(context.accountId(), context.clusterKey(), context.databaseName(),
+ context.iamRoleArns(), !connection.getAutoCommit());
+ try {
+ if (preparation.prepare(sql, session, new RedshiftDataBackendSql(connection))) {
+ stored.status = RedshiftDataStatementStore.Status.FINISHED;
+ return;
+ }
+ } catch (SpectrumSqlException exception) {
+ throw new SQLException(exception.getMessage(), exception.sqlState(), exception);
+ } catch (SpectrumReadException exception) {
+ throw new SQLException(exception.getMessage(), exception.sqlState(), exception);
+ } catch (RuntimeException exception) {
+ // Anything else from Glue or the read path must fail the statement, not leave it running.
+ throw new SQLException("External query preparation failed: " + exception.getMessage(), "58030", exception);
+ }
+ }
ParsedSql parsed = RedshiftDataSqlParameters.parse(sql);
long t0 = System.nanoTime();
try (PreparedStatement statement = connection.prepareStatement(parsed.sql())) {
@@ -309,44 +340,57 @@ public class RedshiftDataService implements Resettable {
long totalDuration = 0;
try (Connection connection = connectionFactory.open(target)) {
- connection.setAutoCommit(false);
- boolean failed = false;
- for (int n = 0; n < sqls.size(); n++) {
- RedshiftDataStatementStore.StoredStatement sub = new RedshiftDataStatementStore.StoredStatement();
- sub.id = parent.id + ":" + (n + 1);
- sub.sql = sqls.get(n);
- sub.sqls = List.of(sqls.get(n));
- sub.clusterIdentifier = parent.clusterIdentifier;
- sub.workgroupName = parent.workgroupName;
- sub.database = parent.database;
- sub.dbUser = parent.dbUser;
- sub.resultFormat = parent.resultFormat;
- Instant now = Instant.now();
- sub.createdAt = now;
- sub.updatedAt = now;
- try {
- runOnConnection(sub, connection, sqls.get(n), Map.of());
- } catch (SQLException e) {
- sub.status = RedshiftDataStatementStore.Status.FAILED;
- sub.error = e.getMessage();
+ BackendSql batchBackend = new RedshiftDataBackendSql(connection);
+ boolean committed = false;
+ try {
+ connection.setAutoCommit(false);
+ boolean failed = false;
+ for (int n = 0; n < sqls.size(); n++) {
+ RedshiftDataStatementStore.StoredStatement sub = new RedshiftDataStatementStore.StoredStatement();
+ sub.id = parent.id + ":" + (n + 1);
+ sub.sql = sqls.get(n);
+ sub.sqls = List.of(sqls.get(n));
+ sub.clusterIdentifier = parent.clusterIdentifier;
+ sub.workgroupName = parent.workgroupName;
+ sub.database = parent.database;
+ sub.dbUser = parent.dbUser;
+ sub.resultFormat = parent.resultFormat;
+ Instant now = Instant.now();
+ sub.createdAt = now;
+ sub.updatedAt = now;
+ try {
+ runOnConnection(sub, connection, sqls.get(n), Map.of(), target.spectrum());
+ } catch (SQLException e) {
+ sub.status = RedshiftDataStatementStore.Status.FAILED;
+ sub.error = e.getMessage();
+ sub.updatedAt = Instant.now();
+ parent.subStatements.add(sub);
+ store.put(sub);
+ totalDuration += sub.durationNanos;
+ parent.status = RedshiftDataStatementStore.Status.FAILED;
+ parent.error = e.getMessage();
+ connection.rollback();
+ failed = true;
+ break;
+ }
sub.updatedAt = Instant.now();
parent.subStatements.add(sub);
store.put(sub);
totalDuration += sub.durationNanos;
- parent.status = RedshiftDataStatementStore.Status.FAILED;
- parent.error = e.getMessage();
- connection.rollback();
- failed = true;
- break;
}
- sub.updatedAt = Instant.now();
- parent.subStatements.add(sub);
- store.put(sub);
- totalDuration += sub.durationNanos;
- }
- if (!failed) {
- connection.commit();
- parent.status = RedshiftDataStatementStore.Status.FINISHED;
+ if (!failed) {
+ connection.commit();
+ committed = true;
+ parent.status = RedshiftDataStatementStore.Status.FINISHED;
+ }
+ } finally {
+ if (preparation != null) {
+ if (committed) {
+ preparation.finishCycle(batchBackend, true);
+ } else {
+ preparation.finishCycle(batchBackend);
+ }
+ }
}
} catch (SQLException e) {
parent.status = RedshiftDataStatementStore.Status.FAILED;
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java
index b1c7076ad8..7465f6c996 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessRuntime.java
@@ -89,6 +89,7 @@ public class RedshiftServerlessRuntime {
String backendId = backendId(region, workgroupName);
proxyManager.stopProxy(relayKey(accountId, backendId));
containerManager.stop(accountId, backendId);
+ proxyManager.forgetSpectrumRuntime(relayKey(accountId, backendId));
credentialBroker.revokeCluster(accountId, backendId);
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessService.java b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessService.java
index 883ace9243..7e8737ccf9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftserverless/RedshiftServerlessService.java
@@ -562,12 +562,20 @@ public class RedshiftServerlessService implements Resettable {
}
return new WorkgroupTarget(workgroup.getWorkgroupArn(), workgroup.getWorkgroupName(),
workgroup.getRuntimeHost(), workgroup.getRuntimePort(), namespace.getDbName(),
- workgroup.getMasterUsername(), namespace.getAdminUserPassword(), adminUserOf(namespace));
+ workgroup.getMasterUsername(), namespace.getAdminUserPassword(), adminUserOf(namespace), namespace.getIamRoles());
}
/** Connection details of a workgroup's backend; {@code adminUsername} is the name clients log in with. */
public record WorkgroupTarget(String arn, String workgroupName, String host, int port, String database,
- String masterUsername, String masterPassword, String adminUsername) {
+ String masterUsername, String masterPassword, String adminUsername, List<String> iamRoleArns) {
+ public WorkgroupTarget(String arn, String workgroupName, String host, int port, String database,
+ String masterUsername, String masterPassword, String adminUsername) {
+ this(arn, workgroupName, host, port, database, masterUsername, masterPassword, adminUsername, List.of());
+ }
+
+ public WorkgroupTarget {
+ iamRoleArns = iamRoleArns == null ? List.of() : List.copyOf(iamRoleArns);
+ }
}
private Workgroup resolveWorkgroup(String nameOrArn, String region) {
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 1ab7b0c624..91f5ae168f 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -65,6 +65,7 @@ quarkus:
- eks/*.json
- aws/*.json
- cloudformation/*.json
+ - redshift/*.sql
- i18n/*.properties
- org/apache/velocity/runtime/defaults/*.properties
# DriverManager finds the JDBC drivers through ServiceLoader, and the MySQL
diff --git a/src/main/resources/redshift/bootstrap-catalog.sql b/src/main/resources/redshift/bootstrap-catalog.sql
index fbeaab20d7..f85fd408bd 100644
--- a/src/main/resources/redshift/bootstrap-catalog.sql
+++ b/src/main/resources/redshift/bootstrap-catalog.sql
@@ -359,6 +359,62 @@ WHERE c.relkind IN ('r', 'm')
AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast');
-- Grant access to all database users (including IAM temporary users)
+CREATE SCHEMA IF NOT EXISTS floci_internal;
+CREATE TABLE IF NOT EXISTS floci_internal.external_catalog (
+ schema_name text PRIMARY KEY,
+ payload jsonb NOT NULL
+);
+
+CREATE OR REPLACE FUNCTION floci_internal.refresh_external_catalog(schema_name text, payload jsonb)
+RETURNS void LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
+ INSERT INTO floci_internal.external_catalog VALUES ($1, $2)
+ ON CONFLICT (schema_name) DO UPDATE SET payload = EXCLUDED.payload;
+$$;
+
+CREATE OR REPLACE FUNCTION floci_internal.purge_external_schema(schema_name text)
+RETURNS void LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
+ DELETE FROM floci_internal.external_catalog WHERE schema_name = $1;
+$$;
+
+CREATE OR REPLACE VIEW pg_catalog.svv_external_schemas AS
+SELECT n.oid AS esoid, 1::smallint AS eskind, n.nspname AS schemaname,
+ n.nspowner::integer AS esowner, c.payload->>'databasename' AS databasename,
+ c.payload->>'esoptions' AS esoptions
+FROM floci_internal.external_catalog c
+JOIN pg_catalog.pg_namespace n ON n.nspname = c.schema_name
+WHERE has_schema_privilege(n.oid, 'USAGE');
+
+CREATE OR REPLACE VIEW pg_catalog.svv_external_tables AS
+SELECT current_database()::text AS redshift_database_name, c.schema_name AS schemaname,
+ t->>'tablename' AS tablename, t->>'tabletype' AS tabletype, t->>'location' AS location,
+ t->>'input_format' AS input_format, t->>'output_format' AS output_format,
+ t->>'serialization_lib' AS serialization_lib, t->>'serde_parameters' AS serde_parameters,
+ (t->>'compressed')::integer AS compressed, t->>'parameters' AS parameters
+FROM floci_internal.external_catalog c
+JOIN pg_catalog.pg_namespace n ON n.nspname = c.schema_name
+CROSS JOIN LATERAL jsonb_array_elements(c.payload->'tables') t
+WHERE has_schema_privilege(n.oid, 'USAGE');
+
+CREATE OR REPLACE VIEW pg_catalog.svv_external_columns AS
+SELECT current_database()::text AS redshift_database_name, c.schema_name AS schemaname,
+ t->>'tablename' AS tablename, t->>'columnname' AS columnname, t->>'external_type' AS external_type,
+ (t->>'columnnum')::integer AS columnnum, (t->>'part_key')::integer AS part_key,
+ t->>'is_nullable' AS is_nullable
+FROM floci_internal.external_catalog c
+JOIN pg_catalog.pg_namespace n ON n.nspname = c.schema_name
+CROSS JOIN LATERAL jsonb_array_elements(c.payload->'columns') t
+WHERE has_schema_privilege(n.oid, 'USAGE');
+
+CREATE OR REPLACE VIEW pg_catalog.svv_external_partitions AS
+SELECT c.schema_name AS schemaname, t->>'tablename' AS tablename, t->>'values' AS values,
+ t->>'location' AS location, t->>'input_format' AS input_format, t->>'output_format' AS output_format,
+ t->>'serialization_lib' AS serialization_lib, t->>'serde_parameters' AS serde_parameters,
+ (t->>'compressed')::integer AS compressed, t->>'parameters' AS parameters
+FROM floci_internal.external_catalog c
+JOIN pg_catalog.pg_namespace n ON n.nspname = c.schema_name
+CROSS JOIN LATERAL jsonb_array_elements(c.payload->'partitions') t
+WHERE has_schema_privilege(n.oid, 'USAGE');
+
GRANT USAGE ON SCHEMA pg_catalog TO PUBLIC;
GRANT ALL ON TABLE pg_catalog.stl_load_errors TO PUBLIC;
GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO PUBLIC;
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java
index 988ddc863e..1dab9c0de8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java
@@ -1,9 +1,19 @@
package io.github.hectorvent.floci.services.redshift;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Database;
import io.github.hectorvent.floci.services.redshift.model.Cluster;
+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatement;
+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatementParser;
+import io.github.hectorvent.floci.services.redshift.spectrum.GlueTableBuilder;
+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalCatalogRegistry;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
import jakarta.inject.Inject;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterEach;
@@ -18,18 +28,36 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Savepoint;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.sql.PreparedStatement;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import java.util.Map;
+import org.postgresql.PGStatement;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@QuarkusTest
+@TestProfile(RedshiftSpectrumIntegrationTest.SpectrumProfile.class)
class RedshiftSpectrumIntegrationTest {
+ public static class SpectrumProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of("quarkus.http.test-port", "17866", "floci.base-url", "http://localhost:17866",
+ "floci.docker.resource-namespace", "spectrum-relational-tests");
+ }
+ }
+
@Inject
RedshiftService redshiftService;
@@ -39,6 +67,12 @@ class RedshiftSpectrumIntegrationTest {
@Inject
IamService iamService;
+ @Inject
+ GlueService glueService;
+
+ @Inject
+ ExternalCatalogRegistry externalRegistry;
+
private static final String ROLE_NAME = "SpectrumItRole";
private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/" + ROLE_NAME;
private static final String TRUST_POLICY = """
@@ -47,6 +81,7 @@ class RedshiftSpectrumIntegrationTest {
private String clusterId;
private String bucket;
+ private String glueDatabase;
@BeforeAll
static void requireDocker() {
@@ -86,6 +121,200 @@ class RedshiftSpectrumIntegrationTest {
s3Service.deleteObject(bucket, "invalid/part-1.csv");
s3Service.deleteBucket(bucket);
}
+ if (glueDatabase != null) {
+ glueService.deleteDatabase(glueDatabase);
+ }
+ }
+
+ @Test
+ @Timeout(120)
+ void joinsExternalAndInternalTablesWithAggregation() throws Exception {
+ bucket = "spectrum-relational-" + System.nanoTime();
+ s3Service.createBucket(bucket, "us-east-1");
+ s3Service.putObject(bucket, "events/part-1.csv", "1,10\n1,5\n2,7\n".getBytes(StandardCharsets.UTF_8),
+ "text/csv", null);
+ glueDatabase = "spectrum_relational_" + System.nanoTime();
+ Database database = new Database();
+ database.setName(glueDatabase);
+ glueService.createDatabase(database);
+ clusterId = "it-relational-" + System.nanoTime();
+ Cluster cluster = createClusterWithRole(clusterId);
+ try (Connection connection = waitForConnection(cluster, "admin", "Secret123");
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE '" + glueDatabase
+ + "' IAM_ROLE '" + ROLE_ARN + "'");
+ String externalDdl = "CREATE EXTERNAL TABLE lake.events (id INTEGER, amount INTEGER) "
+ + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE LOCATION 's3://"
+ + bucket + "/events/'";
+ ExternalStatement.CreateTable external = (ExternalStatement.CreateTable)
+ new ExternalStatementParser().parse(externalDdl).orElseThrow();
+ glueService.createTable(glueDatabase, GlueTableBuilder.toGlueTable(external));
+ statement.execute("CREATE TABLE customers (id INTEGER, country VARCHAR(2))");
+ statement.execute("INSERT INTO customers VALUES (1, 'VN'), (2, 'US')");
+ try (PreparedStatement ddl = connection.prepareStatement("CREATE EXTERNAL SCHEMA rolledback_lake "
+ + "FROM DATA CATALOG DATABASE '" + glueDatabase + "' IAM_ROLE '" + ROLE_ARN + "'")) {
+ ddl.unwrap(PGStatement.class).setPrepareThreshold(1);
+ assertEquals(0, ddl.getParameterMetaData().getParameterCount());
+ connection.setAutoCommit(false);
+ try {
+ SQLException rejected = assertThrows(SQLException.class, ddl::execute);
+ assertEquals("0A000", rejected.getSQLState());
+ } finally {
+ connection.rollback();
+ connection.setAutoCommit(true);
+ }
+ assertTrue(externalRegistry.find("000000000000", "000000000000:" + clusterId,
+ "dev", "rolledback_lake").isEmpty());
+ }
+ connection.setAutoCommit(false);
+ try {
+ SQLException rejected = assertThrows(SQLException.class, () -> statement.execute(
+ externalDdl.replace("lake.events", "lake.in_transaction")));
+ assertEquals("25001", rejected.getSQLState());
+ assertTrue(rejected.getMessage().contains("cannot run inside a transaction block"));
+ } finally {
+ connection.rollback();
+ connection.setAutoCommit(true);
+ }
+ assertThrows(AwsException.class, () -> glueService.getTable(glueDatabase, "in_transaction"));
+ try (PreparedStatement query = connection.prepareStatement("SELECT c.country, SUM(e.amount) AS total "
+ + "FROM lake.events e JOIN customers c ON c.id=e.id WHERE e.amount > ? "
+ + "GROUP BY c.country ORDER BY total DESC")) {
+ query.setInt(1, 0);
+ try (ResultSet rows = query.executeQuery()) {
+ assertTrue(rows.next());
+ assertEquals("VN", rows.getString(1));
+ assertEquals(15L, rows.getLong(2));
+ assertTrue(rows.next());
+ assertEquals("US", rows.getString(1));
+ assertEquals(7L, rows.getLong(2));
+ assertTrue(!rows.next());
+ }
+ query.unwrap(PGStatement.class).setPrepareThreshold(1);
+ statement.execute("CREATE VIEW event_amounts AS SELECT amount FROM lake.events");
+ s3Service.putObject(bucket, "events/part-1.csv", "1,20\n2,7\n".getBytes(StandardCharsets.UTF_8),
+ "text/csv", null);
+ try (ResultSet rows = query.executeQuery()) {
+ assertTrue(rows.next());
+ assertEquals(20L, rows.getLong(2));
+ }
+ s3Service.putObject(bucket, "events/part-1.csv", "1,30\n2,7\n".getBytes(StandardCharsets.UTF_8),
+ "text/csv", null);
+ try (ResultSet rows = query.executeQuery()) {
+ assertTrue(rows.next());
+ assertEquals(30L, rows.getLong(2));
+ }
+ }
+ try (ResultSet rows = statement.executeQuery("SELECT SUM(a.amount) FROM lake.events a "
+ + "JOIN lake.events b ON a.id=b.id HAVING SUM(a.amount)>10 ORDER BY 1 LIMIT 1")) {
+ assertTrue(rows.next());
+ assertEquals(37L, rows.getLong(1));
+ }
+ try (ResultSet rows = statement.executeQuery("SELECT SUM(amount) FROM event_amounts")) {
+ assertTrue(rows.next());
+ assertEquals(37L, rows.getLong(1));
+ }
+ connection.setAutoCommit(false);
+ s3Service.putObject(bucket, "events/part-1.csv", "1,35\n2,7\n".getBytes(StandardCharsets.UTF_8),
+ "text/csv", null);
+ try (PreparedStatement cursor = connection.prepareStatement("SELECT id, amount FROM lake.events ORDER BY id")) {
+ cursor.setFetchSize(1);
+ try (ResultSet rows = cursor.executeQuery()) {
+ assertTrue(rows.next());
+ assertEquals(1, rows.getInt(1));
+ try (PreparedStatement another = connection.prepareStatement("SELECT SUM(amount) FROM lake.events");
+ ResultSet sum = another.executeQuery()) {
+ assertTrue(sum.next());
+ assertEquals(42L, sum.getLong(1));
+ }
+ assertTrue(rows.next());
+ assertEquals(2, rows.getInt(1));
+ assertTrue(!rows.next());
+ }
+ }
+ Savepoint beforeReload = connection.setSavepoint();
+ s3Service.putObject(bucket, "events/part-1.csv", "1,40\n2,7\n".getBytes(StandardCharsets.UTF_8),
+ "text/csv", null);
+ try (ResultSet rows = statement.executeQuery("SELECT SUM(amount) FROM lake.events")) {
+ assertTrue(rows.next());
+ assertEquals(47L, rows.getLong(1));
+ }
+ connection.rollback(beforeReload);
+ try (ResultSet rows = statement.executeQuery("SELECT SUM(amount) FROM lake.events")) {
+ assertTrue(rows.next());
+ assertEquals(47L, rows.getLong(1));
+ }
+ connection.rollback();
+ connection.setAutoCommit(true);
+ try (ResultSet rows = statement.executeQuery("SELECT SUM(amount) FROM lake.events")) {
+ assertTrue(rows.next());
+ assertEquals(47L, rows.getLong(1));
+ }
+ s3Service.putObject(bucket, "events/part-1.csv", "1,30\n2,7\n".getBytes(StandardCharsets.UTF_8),
+ "text/csv", null);
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ String queryId = RestAssuredJsonUtils.awsAction("RedshiftData", "ExecuteStatement", """
+ {"ClusterIdentifier":"%s","DbUser":"admin","Database":"dev",
+ "Sql":"SELECT SUM(amount) AS total FROM lake.events"}
+ """.formatted(clusterId)).then().statusCode(200).extract().path("Id");
+ assertEquals("FINISHED", RestAssuredJsonUtils.awsAction("RedshiftData", "DescribeStatement",
+ "{\"Id\":\"" + queryId + "\"}").then().statusCode(200).extract().path("Status"));
+ Number total = RestAssuredJsonUtils.awsAction("RedshiftData", "GetStatementResult",
+ "{\"Id\":\"" + queryId + "\"}").then().statusCode(200).extract().path("Records[0][0].longValue");
+ assertEquals(37L, total.longValue());
+ iamService.updateAssumeRolePolicy(ROLE_NAME, """
+ {"Version":"2012-10-17","Statement":[{"Effect":"Deny",
+ "Principal":{"Service":"redshift.amazonaws.com"},"Action":"sts:AssumeRole"}]}""");
+ try {
+ SQLException denied = assertThrows(SQLException.class,
+ () -> statement.executeQuery("SELECT SUM(amount) FROM lake.events"));
+ assertEquals("42501", denied.getSQLState());
+ } finally {
+ iamService.updateAssumeRolePolicy(ROLE_NAME, TRUST_POLICY);
+ }
+ statement.execute("CREATE USER spectrum_reader PASSWORD 'Password123'");
+ statement.execute("GRANT USAGE ON SCHEMA lake TO spectrum_reader");
+ statement.execute("GRANT SELECT ON lake.events TO spectrum_reader");
+ try (Connection reader = DriverManager.getConnection("jdbc:postgresql://127.0.0.1:"
+ + cluster.getEndpoint().getPort() + "/dev?socketTimeout=20", "spectrum_reader", "Password123");
+ Statement readerQuery = reader.createStatement();
+ ResultSet rows = readerQuery.executeQuery("SELECT SUM(amount) FROM lake.events")) {
+ assertTrue(rows.next());
+ assertEquals(37L, rows.getLong(1));
+ }
+ s3Service.putObject(bucket, "events/part-1.csv", "1,40\n2,7\n".getBytes(StandardCharsets.UTF_8),
+ "text/csv", null);
+ CountDownLatch ready = new CountDownLatch(2);
+ CountDownLatch start = new CountDownLatch(1);
+ try (ExecutorService queries = Executors.newVirtualThreadPerTaskExecutor()) {
+ Future<Long> first = queries.submit(() -> concurrentTotal(cluster, ready, start));
+ Future<Long> second = queries.submit(() -> concurrentTotal(cluster, ready, start));
+ assertTrue(ready.await(20, TimeUnit.SECONDS));
+ start.countDown();
+ assertEquals(47L, first.get(30, TimeUnit.SECONDS));
+ assertEquals(47L, second.get(30, TimeUnit.SECONDS));
+ }
+ s3Service.putObject(bucket, "events/part-1.csv", "bad,40\n".getBytes(StandardCharsets.UTF_8),
+ "text/csv", null);
+ assertThrows(SQLException.class, () -> statement.executeQuery("SELECT SUM(amount) FROM lake.events"));
+ s3Service.putObject(bucket, "events/part-1.csv", "1,40\n2,7\n".getBytes(StandardCharsets.UTF_8),
+ "text/csv", null);
+ try (ResultSet rows = statement.executeQuery("SELECT SUM(amount) FROM lake.events")) {
+ assertTrue(rows.next());
+ assertEquals(47L, rows.getLong(1));
+ }
+ glueService.deleteTable(glueDatabase, "events");
+ SQLException missing = assertThrows(SQLException.class,
+ () -> statement.executeQuery("SELECT * FROM lake.events"));
+ assertEquals("42P01", missing.getSQLState());
+ try (ResultSet rows = statement.executeQuery("SELECT 1")) {
+ assertTrue(rows.next());
+ }
+ }
+ String deleted = clusterId;
+ redshiftService.deleteCluster(deleted);
+ clusterId = null;
+ assertTrue(externalRegistry.list("000000000000", "000000000000:" + deleted, "dev").isEmpty());
}
@Test
@@ -214,4 +443,16 @@ class RedshiftSpectrumIntegrationTest {
.pollInterval(Duration.ofMillis(500)).ignoreExceptions()
.until(() -> DriverManager.getConnection(url, username, password), Objects::nonNull);
}
+
+ private static long concurrentTotal(Cluster cluster, CountDownLatch ready, CountDownLatch start) throws Exception {
+ try (Connection connection = waitForConnection(cluster, "admin", "Secret123");
+ PreparedStatement query = connection.prepareStatement("SELECT SUM(amount) FROM lake.events")) {
+ ready.countDown();
+ assertTrue(start.await(20, TimeUnit.SECONDS));
+ try (ResultSet rows = query.executeQuery()) {
+ assertTrue(rows.next());
+ return rows.getLong(1);
+ }
+ }
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java
index 3ef223e0f2..ec81fa4257 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/BackendResponseCoordinatorTest.java
@@ -14,6 +14,28 @@ class BackendResponseCoordinatorTest {
private static final byte[] EMPTY_BODY = new byte[0];
+ @Test
+ void beginUpdatesTransactionStateBeforeSync() {
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());
+ coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);
+ coordinator.onBackendFrame('C', new byte[]{'B', 'E', 'G', 'I', 'N', 0});
+ assertEquals('T', coordinator.lastReadyStatus());
+ coordinator.onBackendFrame('C', new byte[]{'R', 'O', 'L', 'L', 'B', 'A', 'C', 'K', 0});
+ assertEquals('T', coordinator.lastReadyStatus());
+ coordinator.onBackendFrame('Z', new byte[]{'I'});
+ assertEquals('I', coordinator.lastReadyStatus());
+ }
+
+ @Test
+ void commitLeavesTransactionBeforeSync() {
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());
+ coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);
+ coordinator.onBackendFrame('C', new byte[]{'B', 'E', 'G', 'I', 'N', 0});
+ coordinator.register(BackendResponseCoordinator.Operation.EXECUTE, null);
+ coordinator.onBackendFrame('C', new byte[]{'C', 'O', 'M', 'M', 'I', 'T', 0});
+ assertEquals('I', coordinator.lastReadyStatus());
+ }
+
@Test
void parseBindAndDescribeCompleteBeforeExecuteBecomesReady() throws InterruptedException {
ExtendedQuerySession session = new ExtendedQuerySession();
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
index 4cb26e418c..83102a0d7f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.redshift.proxy;
import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumInterceptor;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.jboss.logging.Logger;
@@ -11,9 +12,11 @@ import org.mockito.Mockito;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;
+import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
@@ -26,6 +29,18 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
class RedshiftInterceptingBridgeTest {
+ @Test
+ void tracksBeginCommandAcrossFragmentedBackendFrames() {
+ BackendResponseCoordinator coordinator = new BackendResponseCoordinator(new ExtendedQuerySession());
+ RedshiftInterceptingBridge.WireFrameTracker tracker =
+ new RedshiftInterceptingBridge.WireFrameTracker(coordinator::onBackendFrame);
+ byte[] begin = new byte[]{'C', 0, 0, 0, 10, 'B', 'E', 'G', 'I', 'N', 0};
+ tracker.consume(begin, 0, 7);
+ assertEquals('I', coordinator.lastReadyStatus());
+ tracker.consume(begin, 7, 4);
+ assertEquals('T', coordinator.lastReadyStatus());
+ }
+
private static final Logger LOG = Logger.getLogger(RedshiftInterceptingBridgeTest.class);
private ServerSocket clientListener;
@@ -39,6 +54,10 @@ class RedshiftInterceptingBridgeTest {
private S3Service s3Stub;
private void startBridge() throws IOException {
+ startBridge(null);
+ }
+
+ private void startBridge(SpectrumInterceptor spectrumInterceptor) throws IOException {
clientListener = new ServerSocket(0);
testClientEnd = new Socket("localhost", clientListener.getLocalPort());
bridgeClientEnd = clientListener.accept();
@@ -49,7 +68,8 @@ class RedshiftInterceptingBridgeTest {
s3Stub = Mockito.mock(S3Service.class);
RedshiftInterceptingBridge bridge = new RedshiftInterceptingBridge(
- bridgeClientEnd, bridgeBackendEnd, s3Stub, Mockito.mock(IamService.class));
+ bridgeClientEnd, bridgeBackendEnd, s3Stub, Mockito.mock(IamService.class), "000000000000", List.of(),
+ spectrumInterceptor, "000000000000:c", "dev");
bridgeThread = Thread.ofVirtual().name("bridge-under-test").start(bridge::run);
}
@@ -306,6 +326,39 @@ class RedshiftInterceptingBridgeTest {
return result;
}
+ @Test
+ @Timeout(20)
+ void locallyHandledQueryInsideTransactionKeepsTheTransactionStatus() throws Exception {
+ SpectrumInterceptor interceptor = Mockito.mock(SpectrumInterceptor.class);
+ Mockito.when(interceptor.intercept(Mockito.anyString(), Mockito.any(), Mockito.any(Socket.class)))
+ .thenReturn(new SpectrumInterceptor.Decision.Forward(), new SpectrumInterceptor.Decision.Handled());
+ startBridge(interceptor);
+ InputStream fromBridge = testClientEnd.getInputStream();
+
+ testClientEnd.getOutputStream().write(PostgresWireDecoder.encodeQuery("BEGIN"));
+ testClientEnd.getOutputStream().flush();
+ assertEquals('Q', nextForwarded().type());
+ writeBackendFrame('C', cString("BEGIN"));
+ writeBackendFrame('Z', new byte[]{'T'});
+ assertEquals('C', fromBridge.read());
+ skipFrameBody(fromBridge);
+ assertEquals('Z', fromBridge.read());
+ skipFrameBody(fromBridge);
+
+ testClientEnd.getOutputStream().write(PostgresWireDecoder.encodeQuery("CREATE EXTERNAL TABLE lake.t (id INT)"));
+ testClientEnd.getOutputStream().flush();
+ assertEquals('C', fromBridge.read());
+ skipFrameBody(fromBridge);
+ assertEquals('Z', fromBridge.read());
+ fromBridge.readNBytes(4);
+ assertEquals('T', fromBridge.read());
+ }
+
+ private static void skipFrameBody(InputStream in) throws IOException {
+ int length = ByteBuffer.wrap(in.readNBytes(4)).getInt();
+ in.readNBytes(length - 4);
+ }
+
@Test
void pumpsBackendBytesToTheClientUnchanged() throws IOException {
startBridge();
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java
index e2662a87b8..f42ba4594e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java
@@ -39,6 +39,7 @@ import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
@@ -69,6 +70,12 @@ class ExternalTableMaterializerTest {
private static final class RecordingBackend implements BackendSql {
private final List<String> statements = new ArrayList<>();
+ private boolean publicationAllowed = true;
+
+ @Override
+ public boolean permitsCachePublication() {
+ return publicationAllowed;
+ }
@Override
public void execute(String sql) {
@@ -115,6 +122,37 @@ class ExternalTableMaterializerTest {
.thenReturn(new S3Service.ListObjectsResult(List.of(), List.of(), false, null));
}
+ @Test
+ void confirmedExtendedCommitPublishesDefinitionsAndFingerprint() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ backend.publicationAllowed = false;
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ materializer.finishCycle(backend, true);
+ RecordingBackend other = new RecordingBackend();
+ assertEquals(ExternalTableMaterializer.Outcome.CURRENT,
+ materializer.ensureCurrent(other, session(false), BINDING, "events"));
+ forceReload();
+ materializer.ensureCurrent(other, session(false), BINDING, "events");
+ assertThat(other.statements.getLast(), containsString("CREATE TABLE IF NOT EXISTS"));
+ }
+
+ @Test
+ void extendedLoadsAreReusableOnlyWithinTheirOwnCycle() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ backend.publicationAllowed = false;
+ assertEquals(ExternalTableMaterializer.Outcome.LOADED,
+ materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+ assertEquals(ExternalTableMaterializer.Outcome.CURRENT,
+ materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+ RecordingBackend other = new RecordingBackend();
+ other.publicationAllowed = false;
+ assertEquals(ExternalTableMaterializer.Outcome.LOADED,
+ materializer.ensureCurrent(other, session(false), BINDING, "events"));
+ materializer.finishCycle(backend);
+ assertEquals(ExternalTableMaterializer.Outcome.LOADED,
+ materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+ }
+
@Test
void loadsThroughAStagingTableAndCachesUnchangedData() {
Table table = csvTable();
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java
index c0614bb9d4..976e03bcb1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java
@@ -19,6 +19,78 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
class PostgresBackendSessionTest {
+ @Test
+ void extendedFailureDoesNotWaitForIgnoredCleanupBeforeClientSync() throws Exception {
+ try (ServerSocket listener = new ServerSocket(0);
+ Socket backend = new Socket("localhost", listener.getLocalPort())) {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<String> names = executor.submit(() -> {
+ try (Socket socket = listener.accept()) {
+ String first = "";
+ Frame frame;
+ do {
+ frame = readFrame(socket.getInputStream());
+ if (frame.type() == 'P') {
+ first = new String(frame.body(), StandardCharsets.UTF_8).split("\0", 2)[0];
+ }
+ } while (frame.type() != 'H');
+ send(socket.getOutputStream(), 'E', new byte[]{'C', '4', '2', 'P', '0', '1', 0,
+ 'M', 'm', 'i', 's', 's', 'i', 'n', 'g', 0, 0});
+ String second = "";
+ do {
+ frame = readFrame(socket.getInputStream());
+ if (frame.type() == 'P') {
+ second = new String(frame.body(), StandardCharsets.UTF_8).split("\0", 2)[0];
+ }
+ } while (frame.type() != 'H');
+ send(socket.getOutputStream(), 'C', "SELECT 1\0".getBytes(StandardCharsets.UTF_8));
+ send(socket.getOutputStream(), '3', new byte[0]);
+ send(socket.getOutputStream(), '3', new byte[0]);
+ return first.equals(second) ? "reused" : "distinct";
+ }
+ });
+ PostgresExtendedBackendSession sql = new PostgresExtendedBackendSession(backend);
+ assertThrows(SpectrumReadException.class, () -> sql.execute("SELECT missing"));
+ SpectrumReadException cleanup = assertThrows(SpectrumReadException.class,
+ () -> sql.execute("DROP TABLE t"));
+ assertEquals("25P02", cleanup.sqlState());
+ sql.onSync();
+ sql.execute("SELECT 1");
+ assertEquals("distinct", names.get());
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void extendedPreparationClosesOnlyItsNamedObjectsWithoutSyncOrSimpleQuery() throws Exception {
+ try (ServerSocket listener = new ServerSocket(0);
+ Socket backend = new Socket("localhost", listener.getLocalPort())) {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<String> messages = executor.submit(() -> {
+ try (Socket socket = listener.accept()) {
+ StringBuilder types = new StringBuilder();
+ Frame frame;
+ do {
+ frame = readFrame(socket.getInputStream());
+ types.append(frame.type());
+ if (frame.type() == 'P') {
+ assertEquals('f', (char) frame.body()[0]);
+ }
+ } while (frame.type() != 'H');
+ send(socket.getOutputStream(), '1', new byte[0]);
+ send(socket.getOutputStream(), '2', new byte[0]);
+ send(socket.getOutputStream(), 'C', "DO\0".getBytes(StandardCharsets.UTF_8));
+ send(socket.getOutputStream(), '3', new byte[0]);
+ send(socket.getOutputStream(), '3', new byte[0]);
+ return types.toString();
+ }
+ });
+ new PostgresExtendedBackendSession(backend).execute("CREATE TABLE t(id integer); INSERT INTO t VALUES (1)");
+ assertEquals("PBECCH", messages.get());
+ executor.shutdownNow();
+ }
+ }
+
@Test
void copyReadFailureDrainsCopyFailResponseWithoutReplacingOriginalCause() throws Exception {
try (ServerSocket listener = new ServerSocket(0);
@@ -188,4 +260,13 @@ class PostgresBackendSessionTest {
private record Frame(char type, byte[] body) {
}
+
+ @Test
+ void preparationFailsFastAfterTheClientsOwnBackendError() {
+ PostgresExtendedBackendSession session = new PostgresExtendedBackendSession(null);
+ session.onBackendError();
+ SpectrumReadException error = assertThrows(SpectrumReadException.class, () -> session.execute("CREATE TABLE t(id INT)"));
+ assertEquals("25P02", error.sqlState());
+ session.onSync();
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparationTest.java
new file mode 100644
index 0000000000..7a2c5dacba
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparationTest.java
@@ -0,0 +1,167 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.core.common.AwsException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+class SpectrumQueryPreparationTest {
+ private static final SpectrumSession SESSION = new SpectrumSession("000000000000", "000000000000:c",
+ "dev", List.of(), false);
+ private static final ExternalSchemaBinding BINDING = new ExternalSchemaBinding(SESSION.accountId(),
+ SESSION.clusterKey(), "dev", "lake", "analytics", "arn:aws:iam::000000000000:role/R");
+ private ExternalCatalogRegistry registry;
+ private ExternalTableMaterializer materializer;
+ private BackendSql backend;
+ private SpectrumQueryPreparation preparation;
+ private GlueService glue;
+
+ @BeforeEach
+ void setUp() {
+ registry = mock(ExternalCatalogRegistry.class);
+ materializer = mock(ExternalTableMaterializer.class);
+ backend = mock(BackendSql.class);
+ when(registry.list(SESSION.accountId(), SESSION.clusterKey(), "dev")).thenReturn(List.of(BINDING));
+ when(materializer.ensureCurrent(any(), any(), any(), anyString()))
+ .thenReturn(ExternalTableMaterializer.Outcome.CURRENT);
+ glue = mock(GlueService.class);
+ preparation = new SpectrumQueryPreparation(registry, materializer, new ExternalStatementParser(),
+ glue, mock(ExternalMetadataWriter.class));
+ }
+
+ @Test
+ void glueDdlFailureUsesSqlErrorInsteadOfEscapingTheWireHandler() {
+ doThrow(new AwsException("AlreadyExistsException", "table already exists", 400))
+ .when(glue).createTable(anyString(), any());
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () -> preparation.prepare(
+ "CREATE EXTERNAL TABLE lake.sales (id INTEGER) STORED AS TEXTFILE LOCATION 's3://bucket/'",
+ SESSION, backend));
+ assertEquals("42P07", error.sqlState());
+ }
+
+ @Test
+ void transactionalSchemaCreateFailsBeforeChangingEitherStore() {
+ SpectrumSession transaction = new SpectrumSession(SESSION.accountId(), SESSION.clusterKey(),
+ SESSION.databaseName(), List.of(), true);
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () -> preparation.prepare(
+ "CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE 'analytics' IAM_ROLE '"
+ + BINDING.iamRoleArn() + "'", transaction, backend));
+ assertEquals("0A000", error.sqlState());
+ verifyNoInteractions(glue, backend);
+ verify(registry, never()).bind(any());
+ }
+
+ @Test
+ void preparesDistinctExternalReferences() {
+ assertFalse(preparation.prepare("SELECT a.id, SUM(b.amount) FROM lake.sales a JOIN lake.sales b "
+ + "ON a.id=b.id JOIN public.customers c ON c.id=a.id GROUP BY a.id ORDER BY a.id", SESSION, backend));
+ verify(materializer).ensureCurrent(backend, SESSION, BINDING, "sales");
+ }
+
+ @Test
+ void leavesInternalSqlUntouched() {
+ assertFalse(preparation.prepare("SELECT 'lake.sales' FROM public.customers lake", SESSION, backend));
+ verifyNoInteractions(materializer, backend);
+ }
+
+ @Test
+ void rejectsExternalMutationInCte() {
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () -> preparation.prepare(
+ "WITH changed AS (DELETE FROM lake.sales RETURNING *) SELECT * FROM changed", SESSION, backend));
+ assertEquals("0A000", error.sqlState());
+ verifyNoInteractions(materializer, backend);
+ }
+
+ @Test
+ void missingGlueTableDoesNotUseCachedRelation() {
+ when(materializer.ensureCurrent(backend, SESSION, BINDING, "sales"))
+ .thenReturn(ExternalTableMaterializer.Outcome.NOT_EXTERNAL);
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () ->
+ preparation.prepare("SELECT * FROM lake.sales", SESSION, backend));
+ assertEquals("42P01", error.sqlState());
+ }
+
+ @Test
+ void failedSchemaCreateDoesNotPublishBinding() {
+ when(registry.find(anyString(), anyString(), anyString(), anyString())).thenReturn(Optional.empty());
+ doThrow(new SpectrumReadException("58030", "backend unavailable")).when(backend).execute(anyString());
+ assertThrows(SpectrumReadException.class, () -> preparation.prepare(
+ "CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE 'analytics' IAM_ROLE '"
+ + BINDING.iamRoleArn() + "'", SESSION, backend));
+ verify(registry, never()).bind(any());
+ }
+
+ private static final String CREATE_LAKE = "CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE 'analytics' IAM_ROLE '"
+ + BINDING.iamRoleArn() + "'";
+
+ @Test
+ void schemaCreatedInClientTransactionIsBoundOnlyWhenItCommits() {
+ when(registry.find(anyString(), anyString(), anyString(), anyString())).thenReturn(Optional.empty());
+ when(backend.permitsCachePublication()).thenReturn(false);
+ assertTrue(preparation.prepare(CREATE_LAKE, SESSION, backend));
+ verify(registry, never()).bind(any());
+ preparation.finishCycle(backend, true);
+ verify(registry).bind(any());
+ }
+
+ @Test
+ void schemaCreatedInRolledBackTransactionIsNeverBound() {
+ when(registry.find(anyString(), anyString(), anyString(), anyString())).thenReturn(Optional.empty());
+ when(backend.permitsCachePublication()).thenReturn(false);
+ assertTrue(preparation.prepare(CREATE_LAKE, SESSION, backend));
+ preparation.finishCycle(backend);
+ preparation.finishCycle(backend, true);
+ verify(registry, never()).bind(any());
+ }
+
+ private static final SpectrumSession TRANSACTION = new SpectrumSession(SESSION.accountId(), SESSION.clusterKey(),
+ "dev", List.of(), true);
+ private static final String CREATE_SALES =
+ "CREATE EXTERNAL TABLE lake.sales (id INTEGER) STORED AS TEXTFILE LOCATION 's3://bucket/'";
+
+ @Test
+ void transactionalTableCreateIsRejectedBeforeChangingEitherStore() {
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare(CREATE_SALES, TRANSACTION, backend));
+ assertEquals("25001", error.sqlState());
+ assertEquals("CREATE EXTERNAL TABLE cannot run inside a transaction block", error.getMessage());
+ verifyNoInteractions(glue, backend);
+ }
+
+ @Test
+ void transactionalTableCreateInUnboundSchemaIsRejectedAndRoutedToPreparation() {
+ String legacy = "CREATE EXTERNAL TABLE legacy.sales (id INTEGER) STORED AS TEXTFILE LOCATION 's3://bucket/'";
+ assertTrue(preparation.handlesDdl(legacy, TRANSACTION));
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare(legacy, TRANSACTION, backend));
+ assertEquals("25001", error.sqlState());
+ verifyNoInteractions(glue, backend);
+ }
+
+ @Test
+ void tableCreatedOutsideTransactionIsStoredInGlue() {
+ assertTrue(preparation.prepare(CREATE_SALES, SESSION, backend));
+ verify(glue).createTable(eq("analytics"), any());
+ }
+
+ @Test
+ void discardingLoadsOnlyDropsPendingLoads() {
+ preparation.discardPendingLoads(backend);
+ verify(materializer).finishCycle(backend);
+ verifyNoInteractions(glue);
+ }
+
+ @Test
+ void schemaCreatedOnAutocommitBackendIsBoundImmediately() {
+ when(registry.find(anyString(), anyString(), anyString(), anyString())).thenReturn(Optional.empty());
+ when(backend.permitsCachePublication()).thenReturn(true);
+ assertTrue(preparation.prepare(CREATE_LAKE, SESSION, backend));
+ verify(registry).bind(any());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSqlTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSqlTest.java
new file mode 100644
index 0000000000..1827ecc1a3
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataBackendSqlTest.java
@@ -0,0 +1,47 @@
+package io.github.hectorvent.floci.services.redshiftdata;
+
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumReadException;
+import org.junit.jupiter.api.Test;
+import org.postgresql.PGConnection;
+import org.postgresql.copy.CopyManager;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+class RedshiftDataBackendSqlTest {
+ @Test
+ void usesCallerConnectionWithoutCommittingOrClosingIt() throws Exception {
+ Connection connection = mock(Connection.class);
+ Statement statement = mock(Statement.class);
+ PGConnection pg = mock(PGConnection.class);
+ CopyManager copy = mock(CopyManager.class);
+ when(connection.createStatement()).thenReturn(statement);
+ when(connection.unwrap(PGConnection.class)).thenReturn(pg);
+ when(pg.getCopyAPI()).thenReturn(copy);
+ InputStream data = new ByteArrayInputStream(new byte[]{1});
+ when(copy.copyIn("COPY t FROM STDIN", data)).thenReturn(2L);
+ RedshiftDataBackendSql backend = new RedshiftDataBackendSql(connection);
+ backend.execute("CREATE TABLE t(id INTEGER)");
+ assertEquals(2L, backend.copyIn("COPY t FROM STDIN", data));
+ verify(statement).execute("CREATE TABLE t(id INTEGER)");
+ verify(statement).close();
+ verify(connection, never()).commit();
+ verify(connection, never()).rollback();
+ verify(connection, never()).close();
+ }
+
+ @Test
+ void preservesSqlStateOnFailure() throws Exception {
+ Connection connection = mock(Connection.class);
+ when(connection.createStatement()).thenThrow(new SQLException("denied", "42501"));
+ SpectrumReadException error = assertThrows(SpectrumReadException.class,
+ () -> new RedshiftDataBackendSql(connection).execute("SELECT 1"));
+ assertEquals("42501", error.sqlState());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolverTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolverTest.java
index 92204ed42c..4784c5998a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolverTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataResourceResolverTest.java
@@ -69,7 +69,8 @@ class RedshiftDataResourceResolverTest {
private static RedshiftServerlessService.WorkgroupTarget workgroupTarget() {
return new RedshiftServerlessService.WorkgroupTarget(
"arn:aws:redshift-serverless:us-east-1:" + ACCOUNT + ":workgroup/wg-id", "wg-1",
- "127.0.0.1", 55433, "analytics", "root", "Secret123", "root");
+ "127.0.0.1", 55433, "analytics", "root", "Secret123", "root",
+ List.of("arn:aws:iam::" + ACCOUNT + ":role/Spectrum"));
}
@Test
@@ -88,6 +89,10 @@ class RedshiftDataResourceResolverTest {
assertEquals("analytics", target.database());
assertEquals("root", target.user());
assertEquals("Secret123", target.password());
+ assertEquals(ACCOUNT, target.spectrum().accountId());
+ assertEquals(ACCOUNT + ":serverless_us-east-1_wg-1", target.spectrum().clusterKey());
+ assertEquals("analytics", target.spectrum().databaseName());
+ assertEquals(List.of("arn:aws:iam::" + ACCOUNT + ":role/Spectrum"), target.spectrum().iamRoleArns());
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.java
index 4c5354da94..0ee9b680d1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.java
@@ -5,6 +5,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumQueryPreparation;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSession;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumReadException;
import org.h2.Driver;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -14,6 +17,7 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.time.Clock;
import java.util.UUID;
+import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -22,6 +26,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.verify;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.argThat;
class RedshiftDataServiceTest {
@@ -34,6 +42,7 @@ class RedshiftDataServiceTest {
private final ObjectMapper om = new ObjectMapper();
private String jdbcUrl;
private RedshiftDataService service;
+ private SpectrumQueryPreparation preparation;
@BeforeEach
void setUp() throws Exception {
@@ -41,13 +50,15 @@ class RedshiftDataServiceTest {
RedshiftDataResourceResolver resolver = mock(RedshiftDataResourceResolver.class);
when(resolver.resolve(any(), any())).thenReturn(new RedshiftDataResourceResolver.DatabaseTarget(
- "arn:aws:redshift:us-east-1:000000000000:cluster:wh", "127.0.0.1", 5439, "dev", "admin", "x"));
+ "arn:aws:redshift:us-east-1:000000000000:cluster:wh", "127.0.0.1", 5439, "dev", "admin", "x",
+ new SpectrumSession("000000000000", "000000000000:wh", "dev", List.of(), false)));
RedshiftDataConnectionFactory factory = mock(RedshiftDataConnectionFactory.class);
when(factory.open(any())).thenAnswer(inv -> DriverManager.getConnection(jdbcUrl, "sa", ""));
+ preparation = mock(SpectrumQueryPreparation.class);
service = new RedshiftDataService(resolver, factory,
- new RedshiftDataStatementStore(24, Clock.systemUTC()), om);
+ new RedshiftDataStatementStore(24, Clock.systemUTC()), om, preparation);
}
private ObjectNode req(String sql) {
@@ -63,6 +74,24 @@ class RedshiftDataServiceTest {
return om.createObjectNode().put("Id", id);
}
+ @Test
+ void preparationFailureIsStoredAsFailedStatement() {
+ doThrow(new SpectrumReadException("42501", "S3 access denied")).when(preparation)
+ .prepare(eq("SELECT * FROM lake.events"), any(), any());
+ ObjectNode executed = service.executeStatement(req("SELECT * FROM lake.events"), REGION);
+ ObjectNode described = service.describeStatement(idOf(executed.path("Id").asText()));
+ assertEquals("FAILED", described.path("Status").asText());
+ assertEquals("S3 access denied", described.path("Error").asText());
+ }
+
+ @Test
+ void batchPreparationUsesTransactionContext() {
+ ObjectNode request = req("unused");
+ request.putArray("Sqls").add("SELECT 1");
+ service.batchExecuteStatement(request, REGION);
+ verify(preparation).prepare(eq("SELECT 1"), argThat(SpectrumSession::inTransaction), any());
+ }
+
@Test
void aWorkgroupStatementNamesItsWorkgroupAndOmitsTheClusterIdentifier() {
ObjectNode request = om.createObjectNode();