Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(redshift): complete Glue external table lifecycle (#5866)

Vũ Thế Huyc33823bd48cf
diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftRelationalSpectrumTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftRelationalSpectrumTest.java
index ba459377c1..890a24198d 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftRelationalSpectrumTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/io/github/hectorvent/floci/compat/RedshiftRelationalSpectrumTest.java
@@ -90,16 +90,7 @@ class RedshiftRelationalSpectrumTest {
assertEquals(7L, rows.getLong(2));
assertFalse(rows.next());
}
- String id = data.executeStatement(r -> r.clusterIdentifier(clusterId).database("dev")
- .dbUser("admin").sql(sql)).id();
- Instant deadline = Instant.now().plusSeconds(30);
- DescribeStatementResponse status = data.describeStatement(r -> r.id(id));
- while (!status.statusAsString().equals("FINISHED") && Instant.now().isBefore(deadline)) {
- assertNotEquals("FAILED", status.statusAsString(), status.error());
- Thread.sleep(100);
- status = data.describeStatement(r -> r.id(id));
- }
- assertEquals("FINISHED", status.statusAsString(), status.error());
+ String id = executeLifecycle(data, clusterId, sql);
GetStatementResultResponse result = data.getStatementResult(r -> r.id(id));
assertEquals(2L, result.totalNumRows());
assertEquals("VN", result.records().get(0).get(0).stringValue());
@@ -119,6 +110,24 @@ class RedshiftRelationalSpectrumTest {
assertTrue(rows.next());
assertEquals(0L, rows.getLong(1));
}
+ setup.execute("CREATE EXTERNAL TABLE lake.partitioned_events (id INTEGER, amount INTEGER) "
+ + "PARTITIONED BY (dt VARCHAR) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' "
+ + "STORED AS TEXTFILE LOCATION 's3://" + bucket + "/partitioned/'");
+ executeLifecycle(data, clusterId, "ALTER TABLE lake.partitioned_events ADD PARTITION (dt='first') "
+ + "LOCATION 's3://" + bucket + "/events/'");
+ assertEquals("first", glue.getPartitions(r -> r.databaseName(database).tableName("partitioned_events"))
+ .partitions().get(0).values().get(0));
+ try (ResultSet rows = setup.executeQuery("SELECT SUM(amount) FROM lake.partitioned_events")) {
+ assertTrue(rows.next());
+ assertEquals(22L, rows.getLong(1));
+ }
+ executeLifecycle(data, clusterId, "DROP TABLE lake.partitioned_events");
+ assertTrue(glue.getTables(r -> r.databaseName(database)).tableList().stream()
+ .noneMatch(table -> table.name().equals("partitioned_events")));
+ executeLifecycle(data, clusterId, "DROP SCHEMA lake CASCADE");
+ assertEquals("events", glue.getTable(r -> r.databaseName(database).name("events")).table().name());
+ assertEquals("1,10\n1,5\n2,7\n", s3.getObjectAsBytes(r -> r.bucket(bucket).key("events/data.csv"))
+ .asUtf8String());
}
} finally {
if (clusterCreated) {
@@ -140,4 +149,18 @@ class RedshiftRelationalSpectrumTest {
}
}
}
+
+ private static String executeLifecycle(RedshiftDataClient data, String clusterId, String sql) throws InterruptedException {
+ String id = data.executeStatement(r -> r.clusterIdentifier(clusterId).database("dev")
+ .dbUser("admin").sql(sql)).id();
+ Instant deadline = Instant.now().plusSeconds(30);
+ DescribeStatementResponse status = data.describeStatement(r -> r.id(id));
+ while (!status.statusAsString().equals("FINISHED") && Instant.now().isBefore(deadline)) {
+ assertNotEquals("FAILED", status.statusAsString(), status.error());
+ Thread.sleep(100);
+ status = data.describeStatement(r -> r.id(id));
+ }
+ assertEquals("FINISHED", status.statusAsString(), status.error());
+ return id;
+ }
}
diff --git a/docs/services/redshift.md b/docs/services/redshift.md
index 4c94c434cf..6f45e9890f 100644
--- a/docs/services/redshift.md
+++ b/docs/services/redshift.md
@@ -337,6 +337,46 @@ Schemas created without a matching Glue database retain the legacy Phase 1 path
Its single-table query restrictions still apply. PostgreSQL materialization approximates Spectrum
execution and does not reproduce AWS distributed planning or a cross-service transaction snapshot.
+### Glue-backed external table lifecycle
+
+Glue-backed schemas support these statements through the Simple Query protocol, JDBC Extended
+Query, and the Redshift Data API:
+
+```sql
+ALTER TABLE lake.events ADD IF NOT EXISTS
+PARTITION (dt='2026-10-17') LOCATION 's3://analytics/events/dt=2026-10-17/';
+
+DROP TABLE IF EXISTS lake.events;
+DROP SCHEMA IF EXISTS lake CASCADE;
+```
+
+`ADD PARTITION` registers partitions in Glue, preserves the declared partition-key order, refreshes
+`svv_external_partitions`, and invalidates cached rows. Multiple partition clauses are supported.
+The entire partition list is validated before registration. A duplicate partition fails unless
+`IF NOT EXISTS` is specified; that clause keeps the existing partition location unchanged.
+If registration or metadata refresh fails, Floci attempts to remove only the partitions created
+by that statement. A failed cleanup is logged while the original error remains the response.
+
+`DROP TABLE` removes the Glue table and its partitions, drops any materialized PostgreSQL table,
+and refreshes the external catalog views. The default `RESTRICT` preserves the Glue definition
+if a local view depends on its materialized table; `CASCADE` also drops those dependent objects.
+No S3 objects are deleted. `IF EXISTS` also removes a stale materialization if the Glue table
+has already been deleted.
+
+`DROP SCHEMA` unregisters the local external schema and clears its materializations and catalog
+view rows. It preserves the Glue database, Glue tables and partitions, and S3 objects. The default
+`RESTRICT` refuses a schema containing Glue tables, including tables that have never been queried.
+If the Glue database was already deleted, the remaining local schema can still be dropped.
+`CASCADE` removes the local schema and its dependent objects. `DROP EXTERNAL DATABASE` and
+multi-object DROP statements are outside this implementation.
+
+Run these catalog changes outside explicit transactions and Data API batches. Floci rejects them
+with SQLSTATE `25001` before changing Glue or PostgreSQL because it cannot roll back Glue changes
+with a PostgreSQL transaction. The schema transaction restriction is an emulator limitation.
+Glue and PostgreSQL updates do not form a distributed transaction: an infrastructure failure after
+one store changes can require retrying the command. The legacy Phase 1 catalog does not gain these
+lifecycle operations.
+
### COPY from S3
`COPY <table> [(<columns>)] FROM 's3://<bucket>/<keyOrPrefix>' [options]` sent over the Simple
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedSpectrumExchange.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedSpectrumExchange.java
index 9f1e6a34e0..27b7d81771 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedSpectrumExchange.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/ExtendedSpectrumExchange.java
@@ -48,8 +48,8 @@ final class ExtendedSpectrumExchange {
PortalCursor cursor = cursors.get(portalName);
if (cursor == null) {
SpectrumInterceptor.Decision decision = interceptor.execute(plan, accountId, database, backend);
- if (decision instanceof SpectrumInterceptor.Decision.Handled) {
- forward(client, commandComplete("CREATE EXTERNAL"));
+ if (decision instanceof SpectrumInterceptor.Decision.Handled handled) {
+ forward(client, commandComplete(handled.commandTag()));
failed = false;
return;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
index 6615d95845..6edbd375bc 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridge.java
@@ -216,9 +216,9 @@ public class RedshiftInterceptingBridge {
writeSimpleSpectrumError(new SpectrumReadException("58030", "Unable to prepare external query"));
return;
}
- if (intercepted && decision[0] instanceof SpectrumInterceptor.Decision.Handled) {
+ if (intercepted && decision[0] instanceof SpectrumInterceptor.Decision.Handled handled) {
coordinator.register(BackendResponseCoordinator.Operation.SIMPLE_QUERY, null);
- write(client.getOutputStream(), commandComplete("CREATE EXTERNAL\0"));
+ write(client.getOutputStream(), commandComplete(handled.commandTag() + "\0"));
reportLocalReady();
return;
}
@@ -270,6 +270,22 @@ public class RedshiftInterceptingBridge {
// resolve to the stale Spectrum plan in handleBind/handleDescribe/handleExecute.
spectrumStatements.remove(parse.statementName());
relationalStatements.remove(parse.statementName());
+ if (spectrumInterceptor != null && parse.parameterTypeOids().isEmpty()
+ && !coordinator.isDiscardingUntilSync()) {
+ try {
+ SpectrumInterceptor.Plan plan = spectrumInterceptor.plan(parse.sql(), spectrumSession());
+ if (plan instanceof SpectrumInterceptor.Plan.Ddl || plan instanceof SpectrumInterceptor.Plan.Query
+ || plan instanceof SpectrumInterceptor.Plan.ExternalDdl) {
+ awaitPriorBackendResponses();
+ spectrumStatements.put(parse.statementName(), plan);
+ write(client.getOutputStream(), backendFrame('1', EMPTY_BODY));
+ return;
+ }
+ } catch (RuntimeException exception) {
+ writeParseSpectrumError(exception);
+ return;
+ }
+ }
// After a backend error the backend discards everything up to Sync, so a preparation round trip
// would only wait out the socket timeout; forward the Parse and let the backend discard it.
if (spectrumInterceptor != null && spectrumInterceptor.preparation() != null
@@ -291,21 +307,6 @@ public class RedshiftInterceptingBridge {
return;
}
}
- if (spectrumInterceptor != null && parse.parameterTypeOids().isEmpty()) {
- try {
- SpectrumInterceptor.Plan plan = spectrumInterceptor.plan(parse.sql(), spectrumSession());
- if (plan instanceof SpectrumInterceptor.Plan.Ddl || plan instanceof SpectrumInterceptor.Plan.Query
- || plan instanceof SpectrumInterceptor.Plan.ExternalDdl) {
- awaitPriorBackendResponses();
- spectrumStatements.put(parse.statementName(), plan);
- write(client.getOutputStream(), backendFrame('1', EMPTY_BODY));
- return;
- }
- } catch (RuntimeException exception) {
- writeParseSpectrumError(exception);
- return;
- }
- }
CopyStatementParser.S3Statement statement = parse.parameterTypeOids().isEmpty()
? parseS3Statement(parse.sql()) : null;
ExtendedQuerySession.Mutation mutation = session.stageParse(parse.statementName(), statement);
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogLocks.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogLocks.java
new file mode 100644
index 0000000000..f2a975ff89
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogLocks.java
@@ -0,0 +1,64 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import jakarta.enterprise.context.ApplicationScoped;
+
+import java.time.Duration;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/** Coordinates Spectrum catalog mutations with readers, including aliases of the same Glue database. */
+@ApplicationScoped
+public class ExternalCatalogLocks {
+ private static final Duration DEFAULT_TRANSACTION_TIMEOUT = Duration.ofSeconds(30);
+
+ private final Duration transactionTimeout;
+ private final Map<CatalogKey, ReentrantReadWriteLock> locks = new ConcurrentHashMap<>();
+
+ public ExternalCatalogLocks() {
+ this(DEFAULT_TRANSACTION_TIMEOUT);
+ }
+
+ public ExternalCatalogLocks(Duration transactionTimeout) {
+ this.transactionTimeout = transactionTimeout;
+ }
+
+ public Guard read(ExternalSchemaBinding binding, boolean transactionOpen) {
+ return acquire(binding, false, transactionOpen);
+ }
+
+ public Guard write(ExternalSchemaBinding binding) {
+ return acquire(binding, true, false);
+ }
+
+ private Guard acquire(ExternalSchemaBinding binding, boolean write, boolean transactionOpen) {
+ CatalogKey key = new CatalogKey(binding.accountId(), binding.glueDatabase().toLowerCase(Locale.ROOT));
+ ReentrantReadWriteLock catalog = locks.computeIfAbsent(key, ignored -> new ReentrantReadWriteLock(true));
+ Lock lock = write ? catalog.writeLock() : catalog.readLock();
+ try {
+ if (transactionOpen) {
+ if (!lock.tryLock(transactionTimeout.toNanos(), TimeUnit.NANOSECONDS)) {
+ throw new SpectrumReadException("58030", "Timed out waiting for external catalog access");
+ }
+ } else {
+ lock.lockInterruptibly();
+ }
+ return new Guard(lock);
+ } catch (InterruptedException exception) {
+ Thread.currentThread().interrupt();
+ throw new SpectrumReadException("58030", "Interrupted while waiting for external catalog access", exception);
+ }
+ }
+
+ private record CatalogKey(String accountId, String databaseName) { }
+
+ public record Guard(Lock lock) implements AutoCloseable {
+ @Override
+ public void close() {
+ lock.unlock();
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParser.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParser.java
index a4a41bf237..27212cf761 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParser.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParser.java
@@ -294,9 +294,22 @@ public class ExternalStatementParser {
return List.of();
}
Map<String, String> values = new LinkedHashMap<>();
- Matcher value = PARTITION_VALUE.matcher(text.substring(open + 1, close).trim() + ",");
- while (value.find()) {
- values.put(identifier(value.group(1)), value.group(2).replace("''", "'"));
+ String keys = text.substring(open + 1, close).trim();
+ Matcher value = PARTITION_VALUE.matcher(keys);
+ int position = 0;
+ while (position < keys.length()) {
+ if (!value.region(position, keys.length()).lookingAt()) {
+ throw new SpectrumSqlException("42601", "Malformed partition key list");
+ }
+ String name = identifier(value.group(1));
+ if (values.keySet().stream().anyMatch(existing -> existing.equalsIgnoreCase(name))) {
+ throw new SpectrumSqlException("42601", "Duplicate partition key: " + name);
+ }
+ values.put(name, value.group(2).replace("''", "'"));
+ position = value.end();
+ if (position == keys.length() && keys.endsWith(",")) {
+ throw new SpectrumSqlException("42601", "Malformed partition key list");
+ }
}
if (values.isEmpty()) {
return List.of();
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java
index 64ff79c87d..c470d52963 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializer.java
@@ -19,6 +19,7 @@ import io.github.hectorvent.floci.services.redshift.proxy.S3CopySimulator;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.io.IOException;
@@ -60,6 +61,7 @@ public class ExternalTableMaterializer {
private final S3Service s3Service;
private final IamService iamService;
private final EmulatorConfig config;
+ private final ExternalCatalogLocks catalogLocks;
private final ConcurrentHashMap<String, String> fingerprints = new ConcurrentHashMap<>();
/** Column definitions each loaded table was created with, to tell a data reload from a schema change. */
private final ConcurrentHashMap<String, String> schemaSignatures = new ConcurrentHashMap<>();
@@ -94,15 +96,34 @@ public class ExternalTableMaterializer {
public ExternalTableMaterializer(FlociDuckClient duckClient, GlueService glueService, S3Service s3Service,
IamService iamService, EmulatorConfig config) {
+ this(duckClient, glueService, s3Service, iamService, config, new ExternalCatalogLocks());
+ }
+
+ @Inject
+ public ExternalTableMaterializer(FlociDuckClient duckClient, GlueService glueService, S3Service s3Service,
+ IamService iamService, EmulatorConfig config, ExternalCatalogLocks catalogLocks) {
this.duckClient = duckClient;
this.glueService = glueService;
this.s3Service = s3Service;
this.iamService = iamService;
this.config = config;
+ this.catalogLocks = catalogLocks;
+ }
+
+ ExternalCatalogLocks catalogLocks() {
+ return catalogLocks;
}
public Outcome ensureCurrent(BackendSql backend, SpectrumSession session, ExternalSchemaBinding binding,
String tableName) {
+ try (ExternalCatalogLocks.Guard ignored = catalogLocks.read(binding,
+ session.inTransaction() || !backend.permitsCachePublication())) {
+ return ensureCurrentLocked(backend, session, binding, tableName);
+ }
+ }
+
+ private Outcome ensureCurrentLocked(BackendSql backend, SpectrumSession session, ExternalSchemaBinding binding,
+ String tableName) {
Table table;
try {
table = RequestScopes.callAs(session.accountId(), () -> glueService.getTable(binding.glueDatabase(), tableName));
@@ -211,6 +232,20 @@ public class ExternalTableMaterializer {
forgetMatching(key -> key.equals(clusterKey + "|" + databaseName + "|" + schemaName + "|" + tableName));
}
+ /** Reloads data after a catalog change without dropping the table's grants and dependent views. */
+ public void invalidateTable(String clusterKey, String databaseName, String schemaName, String tableName) {
+ String key = clusterKey + "|" + databaseName + "|" + schemaName + "|" + tableName;
+ ReentrantLock lock = locks.computeIfAbsent(key, ignored -> new ReentrantLock());
+ lock.lock();
+ try {
+ fingerprints.remove(key);
+ latestLoads.remove(key);
+ pendingLoads.values().forEach(loads -> loads.remove(key));
+ } finally {
+ lock.unlock();
+ }
+ }
+
/** Forgets every table of one external schema, so a dropped and recreated schema is reloaded. */
public void forgetSchema(String clusterKey, String databaseName, String schemaName) {
String prefix = schemaPrefix(clusterKey, databaseName, schemaName);
@@ -218,9 +253,19 @@ public class ExternalTableMaterializer {
}
private void forgetMatching(Predicate<String> matches) {
- fingerprints.keySet().removeIf(matches);
- schemaSignatures.keySet().removeIf(matches);
- latestLoads.keySet().removeIf(matches);
+ locks.forEach((key, lock) -> {
+ if (matches.test(key)) {
+ lock.lock();
+ try {
+ fingerprints.remove(key);
+ schemaSignatures.remove(key);
+ latestLoads.remove(key);
+ pendingLoads.values().forEach(loads -> loads.remove(key));
+ } finally {
+ lock.unlock();
+ }
+ }
+ });
// Keep locks: removing a held lock lets another thread create a second lock for the same table.
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java
index afa68e92ed..caf1badd40 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumInterceptor.java
@@ -89,7 +89,7 @@ public final class SpectrumInterceptor {
switch (plan) {
case Plan.ExternalDdl(String sql, SpectrumSession session) -> {
if (preparation.prepare(sql, session, new PostgresBackendSession(backend))) {
- return new Decision.Handled();
+ return new Decision.Handled(preparation.commandTag(sql));
}
return execute(plan(sql, accountId, databaseName), accountId, databaseName, backend);
}
@@ -145,7 +145,10 @@ public final class SpectrumInterceptor {
}
public sealed interface Decision permits Decision.Handled, Decision.Forward, Decision.Rewritten {
- record Handled() implements Decision {
+ record Handled(String commandTag) implements Decision {
+ public Handled() {
+ this("CREATE EXTERNAL");
+ }
}
record Forward() implements Decision {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparation.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparation.java
index 8a4c3a0d3b..06d1d3bf82 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparation.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumQueryPreparation.java
@@ -4,9 +4,13 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.services.glue.GlueService;
import io.github.hectorvent.floci.services.glue.model.Database;
+import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.Table;
import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
@@ -24,15 +28,24 @@ public class SpectrumQueryPreparation {
private final ExternalStatementParser parser;
private final GlueService glue;
private final ExternalMetadataWriter metadata;
+ private final ExternalCatalogLocks catalogLocks;
private final Map<BackendSql, List<ExternalSchemaBinding>> pendingBindings = new ConcurrentHashMap<>();
public SpectrumQueryPreparation(ExternalCatalogRegistry registry, ExternalTableMaterializer materializer,
ExternalStatementParser parser, GlueService glue, ExternalMetadataWriter metadata) {
+ this(registry, materializer, parser, glue, metadata, new ExternalCatalogLocks());
+ }
+
+ @Inject
+ public SpectrumQueryPreparation(ExternalCatalogRegistry registry, ExternalTableMaterializer materializer,
+ ExternalStatementParser parser, GlueService glue, ExternalMetadataWriter metadata,
+ ExternalCatalogLocks catalogLocks) {
this.registry = registry;
this.materializer = materializer;
this.parser = parser;
this.glue = glue;
this.metadata = metadata;
+ this.catalogLocks = catalogLocks;
}
public boolean prepare(String sql, SpectrumSession session, BackendSql backend) {
@@ -57,10 +70,24 @@ public class SpectrumQueryPreparation {
}
public boolean handlesDdl(String sql, SpectrumSession session) {
- Optional<ExternalStatement> statement = parser.parse(sql);
- return statement.orElse(null) instanceof ExternalStatement.CreateSchema
- || statement.orElse(null) instanceof ExternalStatement.CreateTable table
- && (session.inTransaction() || bindings(session).containsKey(table.schemaName()));
+ ExternalStatement statement = parser.parse(sql).orElse(null);
+ return switch (statement) {
+ case ExternalStatement.CreateSchema ignored -> true;
+ case ExternalStatement.CreateTable table -> session.inTransaction() || bindings(session).containsKey(table.schemaName());
+ case ExternalStatement.AddPartitions partitions -> bindings(session).containsKey(partitions.schemaName());
+ case ExternalStatement.DropTable table -> bindings(session).containsKey(table.schemaName());
+ case ExternalStatement.DropSchema schema -> bindings(session).containsKey(schema.schemaName());
+ case null -> false;
+ };
+ }
+
+ public String commandTag(String sql) {
+ return switch (parser.parse(sql).orElse(null)) {
+ case ExternalStatement.AddPartitions ignored -> "ALTER TABLE";
+ case ExternalStatement.DropTable ignored -> "DROP TABLE";
+ case ExternalStatement.DropSchema ignored -> "DROP SCHEMA";
+ case null, default -> "CREATE EXTERNAL";
+ };
}
private boolean prepareInScope(String sql, SpectrumSession session, BackendSql backend) {
@@ -75,10 +102,31 @@ public class SpectrumQueryPreparation {
}
if (statement.orElse(null) instanceof ExternalStatement.CreateTable table && bindings.containsKey(table.schemaName())) {
ExternalSchemaBinding binding = bindings.get(table.schemaName());
- glue.createTable(binding.glueDatabase(), GlueTableBuilder.toGlueTable(table));
- metadata.refresh(backend, session.accountId(), binding);
+ try (ExternalCatalogLocks.Guard ignored = catalogLocks.write(binding)) {
+ requireCurrentBinding(session, binding);
+ glue.createTable(binding.glueDatabase(), GlueTableBuilder.toGlueTable(table));
+ metadata.refresh(backend, session.accountId(), binding);
+ }
return true;
}
+ switch (statement.orElse(null)) {
+ case ExternalStatement.AddPartitions partitions when bindings.containsKey(partitions.schemaName()) -> {
+ rejectTransaction(session);
+ addPartitions(partitions, bindings.get(partitions.schemaName()), session, backend);
+ return true;
+ }
+ case ExternalStatement.DropTable table when bindings.containsKey(table.schemaName()) -> {
+ rejectTransaction(session);
+ dropTable(table, bindings.get(table.schemaName()), session, backend);
+ return true;
+ }
+ case ExternalStatement.DropSchema schema when bindings.containsKey(schema.schemaName()) -> {
+ rejectTransaction(session);
+ dropSchema(schema, bindings.get(schema.schemaName()), session, backend);
+ return true;
+ }
+ case null, default -> { }
+ }
Optional<ExternalReferenceScanner.Reference> write = ExternalReferenceScanner.writeTarget(sql, bindings.keySet());
if (write.isPresent()) {
throw new SpectrumSqlException("0A000", "Writes to external relation \""
@@ -87,18 +135,155 @@ public class SpectrumQueryPreparation {
Set<String> refreshed = new HashSet<>();
for (ExternalReferenceScanner.Reference reference : ExternalReferenceScanner.scan(sql, bindings.keySet())) {
ExternalSchemaBinding binding = bindings.get(reference.schema());
- ExternalTableMaterializer.Outcome outcome = materializer.ensureCurrent(backend, session, binding, reference.table());
- if (outcome == ExternalTableMaterializer.Outcome.NOT_EXTERNAL) {
- throw new SpectrumSqlException("42P01", "external relation \"" + reference.schema()
- + "." + reference.table() + "\" does not exist");
- }
- if (outcome == ExternalTableMaterializer.Outcome.LOADED && refreshed.add(binding.schemaName())) {
- metadata.refresh(backend, session.accountId(), binding);
+ try (ExternalCatalogLocks.Guard ignored = catalogLocks.read(binding,
+ session.inTransaction() || !backend.permitsCachePublication())) {
+ requireCurrentBinding(session, binding);
+ ExternalTableMaterializer.Outcome outcome = materializer.ensureCurrent(backend, session, binding, reference.table());
+ if (outcome == ExternalTableMaterializer.Outcome.NOT_EXTERNAL) {
+ throw new SpectrumSqlException("42P01", "external relation \"" + reference.schema()
+ + "." + reference.table() + "\" does not exist");
+ }
+ if (outcome == ExternalTableMaterializer.Outcome.LOADED && refreshed.add(binding.schemaName())) {
+ metadata.refresh(backend, session.accountId(), binding);
+ }
}
}
return false;
}
+ private static void rejectTransaction(SpectrumSession session) {
+ if (session.inTransaction()) {
+ throw new SpectrumSqlException("25001", "External catalog changes cannot run inside a transaction block");
+ }
+ }
+
+ private void addPartitions(ExternalStatement.AddPartitions statement, ExternalSchemaBinding binding,
+ SpectrumSession session, BackendSql backend) {
+ try (ExternalCatalogLocks.Guard ignored = catalogLocks.write(binding)) {
+ requireCurrentBinding(session, binding);
+ addPartitionsLocked(statement, binding, session, backend);
+ }
+ }
+
+ private void addPartitionsLocked(ExternalStatement.AddPartitions statement, ExternalSchemaBinding binding,
+ SpectrumSession session, BackendSql backend) {
+ Table table = glue.getTable(binding.glueDatabase(), statement.tableName());
+ if (table.getPartitionKeys() == null || table.getPartitionKeys().isEmpty()) {
+ throw new SpectrumSqlException("22023", "External table has no partition keys");
+ }
+ List<Partition> requested = statement.partitions().stream()
+ .map(spec -> GlueTableBuilder.toGluePartition(table, spec)).toList();
+ Set<List<String>> values = new HashSet<>();
+ for (Partition partition : glue.getPartitions(binding.glueDatabase(), statement.tableName())) {
+ values.add(partition.getValues());
+ }
+ List<Partition> additions = new ArrayList<>();
+ for (Partition partition : requested) {
+ if (values.add(partition.getValues())) {
+ additions.add(partition);
+ } else if (!statement.ifNotExists()) {
+ throw new SpectrumSqlException("42710", "Partition already exists: " + partition.getValues());
+ }
+ }
+ requireSchemaOwner(backend, binding.schemaName());
+ List<Partition> created = new ArrayList<>();
+ try {
+ for (Partition partition : additions) {
+ glue.createPartition(binding.glueDatabase(), statement.tableName(), partition);
+ created.add(partition);
+ }
+ metadata.refresh(backend, session.accountId(), binding);
+ } catch (RuntimeException failure) {
+ for (Partition partition : created.reversed()) {
+ try {
+ glue.deletePartition(binding.glueDatabase(), statement.tableName(), partition.getValues());
+ } catch (RuntimeException cleanupFailure) {
+ failure.addSuppressed(cleanupFailure);
+ LOG.warnv(cleanupFailure, "Unable to undo partition {0} of {1}.{2}",
+ partition.getValues(), binding.glueDatabase(), statement.tableName());
+ }
+ }
+ throw failure;
+ } finally {
+ materializer.invalidateTable(session.clusterKey(), session.databaseName(), binding.schemaName(), statement.tableName());
+ }
+ }
+
+ private void dropTable(ExternalStatement.DropTable statement, ExternalSchemaBinding binding,
+ SpectrumSession session, BackendSql backend) {
+ try (ExternalCatalogLocks.Guard ignored = catalogLocks.write(binding)) {
+ requireCurrentBinding(session, binding);
+ dropTableLocked(statement, binding, session, backend);
+ }
+ }
+
+ private void dropTableLocked(ExternalStatement.DropTable statement, ExternalSchemaBinding binding,
+ SpectrumSession session, BackendSql backend) {
+ boolean exists = true;
+ try {
+ glue.getTable(binding.glueDatabase(), statement.tableName());
+ } catch (AwsException exception) {
+ if (!statement.ifExists() || !"EntityNotFoundException".equals(exception.getErrorCode())) {
+ throw exception;
+ }
+ exists = false;
+ }
+ requireSchemaOwner(backend, binding.schemaName());
+ // Cold tables have no PostgreSQL relation. Check any cached relation's dependencies before deleting Glue metadata.
+ backend.execute("DROP TABLE IF EXISTS " + ExternalTableMaterializer.quote(binding.schemaName()) + "."
+ + ExternalTableMaterializer.quote(statement.tableName()) + (statement.cascade() ? " CASCADE" : " RESTRICT"));
+ materializer.forget(session.clusterKey(), session.databaseName(), binding.schemaName(), statement.tableName());
+ if (exists) {
+ glue.deleteTable(binding.glueDatabase(), statement.tableName());
+ }
+ metadata.refresh(backend, session.accountId(), binding);
+ }
+
+ private void dropSchema(ExternalStatement.DropSchema statement, ExternalSchemaBinding binding,
+ SpectrumSession session, BackendSql backend) {
+ try (ExternalCatalogLocks.Guard ignored = catalogLocks.write(binding)) {
+ requireCurrentBinding(session, binding);
+ requireSchemaOwner(backend, binding.schemaName());
+ if (!statement.cascade() && hasGlueTables(binding)) {
+ throw new SpectrumSqlException("2BP01", "Cannot drop external schema because it contains external tables");
+ }
+ // One implicit PostgreSQL transaction keeps the local schema and its SVV metadata in sync.
+ backend.execute("DROP SCHEMA " + (statement.ifExists() ? "IF EXISTS " : "")
+ + ExternalTableMaterializer.quote(binding.schemaName()) + (statement.cascade() ? " CASCADE" : " RESTRICT")
+ + "; " + metadata.purgeSql(binding.schemaName()));
+ registry.unbind(session.accountId(), session.clusterKey(), session.databaseName(), binding.schemaName());
+ materializer.forgetSchema(session.clusterKey(), session.databaseName(), binding.schemaName());
+ }
+ }
+
+ private boolean hasGlueTables(ExternalSchemaBinding binding) {
+ try {
+ return !glue.getTables(binding.glueDatabase()).isEmpty();
+ } catch (AwsException exception) {
+ if (!"EntityNotFoundException".equals(exception.getErrorCode())) {
+ throw exception;
+ }
+ return false;
+ }
+ }
+
+ private void requireCurrentBinding(SpectrumSession session, ExternalSchemaBinding binding) {
+ if (!binding.equals(bindings(session).get(binding.schemaName()))) {
+ throw new SpectrumSqlException("42P01", "External schema binding changed; retry the statement");
+ }
+ }
+
+ private static void requireSchemaOwner(BackendSql backend, String schemaName) {
+ String body = "BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace n WHERE n.nspname = "
+ + sqlLiteral(schemaName) + " AND pg_catalog.pg_has_role(current_user, n.nspowner, 'USAGE')) THEN "
+ + "RAISE EXCEPTION 'Must be owner of external schema' USING ERRCODE = '42501'; END IF; END";
+ backend.execute("DO " + sqlLiteral(body));
+ }
+
+ private static String sqlLiteral(String value) {
+ return "'" + value.replace("'", "''") + "'";
+ }
+
private boolean createSchema(ExternalStatement.CreateSchema schema, SpectrumSession session, BackendSql backend) {
if (session.inTransaction()) {
throw new SpectrumSqlException("0A000", "CREATE EXTERNAL SCHEMA inside a transaction is not supported");
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java
index f685aec556..c467bf0ec5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftSpectrumIntegrationTest.java
@@ -21,6 +21,8 @@ import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.postgresql.PGStatement;
import java.nio.charset.StandardCharsets;
@@ -438,6 +440,125 @@ class RedshiftSpectrumIntegrationTest {
}
}
+ @ParameterizedTest
+ @ValueSource(strings = {"simple", "extended"})
+ @Timeout(120)
+ void externalLifecycleKeepsGlueCatalogAndCachedRowsConsistent(String queryMode) throws Exception {
+ bucket = "spectrum-lifecycle-" + System.nanoTime();
+ glueDatabase = "lifecycle_" + System.nanoTime();
+ clusterId = "lifecycle-" + System.nanoTime();
+ s3Service.createBucket(bucket, "us-east-1");
+ s3Service.putObject(bucket, "events/part-1.csv", "1,10\n2,20\n".getBytes(StandardCharsets.UTF_8), "text/csv", null);
+ s3Service.putObject(bucket, "invalid/part-1.csv", "3,7\n".getBytes(StandardCharsets.UTF_8), "text/csv", null);
+ Database database = new Database();
+ database.setName(glueDatabase);
+ glueService.createDatabase(database);
+ Cluster cluster = createClusterWithRole(clusterId);
+ String url = "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort()
+ + "/dev?preferQueryMode=" + queryMode + "&socketTimeout=30";
+ String create = "CREATE EXTERNAL TABLE lake.events (id INTEGER, amount INTEGER) PARTITIONED BY (dt VARCHAR) "
+ + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE LOCATION 's3://" + bucket + "/root/'";
+ String firstPartition = "ALTER TABLE lake.events ADD PARTITION (dt='first') LOCATION 's3://" + bucket + "/events/'";
+ try (Connection connection = DriverManager.getConnection(url, "admin", "Secret123");
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE '" + glueDatabase
+ + "' IAM_ROLE '" + ROLE_ARN + "'");
+ statement.execute(create);
+ statement.execute(create.replace("lake.events", "lake.cold_events"));
+ statement.execute("CREATE USER lifecycle_reader PASSWORD 'Password123'");
+ statement.execute("GRANT USAGE ON SCHEMA lake TO lifecycle_reader");
+ try (Connection reader = DriverManager.getConnection(url, "lifecycle_reader", "Password123");
+ Statement readOnly = reader.createStatement()) {
+ for (String deniedSql : List.of(firstPartition.replace("lake.events", "lake.cold_events"),
+ "DROP TABLE lake.cold_events", "DROP SCHEMA lake RESTRICT")) {
+ SQLException denied = assertThrows(SQLException.class, () -> readOnly.execute(deniedSql));
+ assertEquals("42501", denied.getSQLState());
+ }
+ }
+ assertTrue(glueService.getPartitions(glueDatabase, "cold_events").isEmpty());
+ statement.execute("DROP TABLE lake.cold_events");
+ assertTotal(statement, 0);
+ statement.execute(firstPartition);
+ assertTotal(statement, 30);
+ assertEquals(List.of("first"), glueService.getPartitions(glueDatabase, "events").getFirst().getValues());
+ statement.execute(firstPartition.replace("ADD PARTITION", "ADD IF NOT EXISTS PARTITION")
+ .replace("/events/'", "/invalid/'"));
+ assertTotal(statement, 30);
+ SQLException duplicate = assertThrows(SQLException.class, () -> statement.execute(firstPartition));
+ assertEquals("42710", duplicate.getSQLState());
+ statement.execute("CREATE VIEW saved_events AS SELECT id FROM lake.events");
+ statement.execute("GRANT SELECT ON lake.events TO lifecycle_reader");
+ lifecycleDataStatement("ALTER TABLE lake.events ADD PARTITION (dt='second') LOCATION 's3://"
+ + bucket + "/invalid/'");
+ assertTotal(statement, 37);
+ try (Connection reader = DriverManager.getConnection(url, "lifecycle_reader", "Password123");
+ Statement readOnly = reader.createStatement();
+ ResultSet rows = readOnly.executeQuery("SELECT COUNT(*) FROM lake.events")) {
+ assertTrue(rows.next());
+ assertEquals(3, rows.getInt(1));
+ SQLException denied = assertThrows(SQLException.class, () -> readOnly.execute("DROP TABLE lake.events"));
+ assertEquals("42501", denied.getSQLState());
+ }
+ try (ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM svv_external_partitions WHERE schemaname='lake'")) {
+ assertTrue(rows.next());
+ assertEquals(2, rows.getInt(1));
+ }
+ connection.setAutoCommit(false);
+ SQLException transaction = assertThrows(SQLException.class, () -> statement.execute("DROP TABLE lake.events"));
+ assertEquals("25001", transaction.getSQLState());
+ connection.rollback();
+ connection.setAutoCommit(true);
+ assertEquals("events", glueService.getTable(glueDatabase, "events").getName());
+
+ SQLException dependent = assertThrows(SQLException.class, () -> statement.execute("DROP TABLE lake.events"));
+ assertEquals("2BP01", dependent.getSQLState());
+ assertEquals("events", glueService.getTable(glueDatabase, "events").getName());
+ statement.execute("DROP TABLE lake.events CASCADE");
+ assertThrows(AwsException.class, () -> glueService.getTable(glueDatabase, "events"));
+ statement.execute("DROP TABLE IF EXISTS lake.events");
+ lifecycleDataStatement(create);
+ lifecycleDataStatement(firstPartition);
+ assertTotal(statement, 30);
+
+ SQLException restrict = assertThrows(SQLException.class, () -> statement.execute("DROP SCHEMA lake RESTRICT"));
+ assertEquals("2BP01", restrict.getSQLState());
+ lifecycleDataStatement("DROP SCHEMA lake CASCADE");
+ assertTrue(externalRegistry.list("000000000000", "000000000000:" + clusterId, "dev").isEmpty());
+ assertEquals("events", glueService.getTable(glueDatabase, "events").getName());
+ assertEquals(1, glueService.getPartitions(glueDatabase, "events").size());
+ assertEquals(2, s3Service.listObjects(bucket, "", null, 100).size());
+ try (ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM svv_external_schemas WHERE schemaname='lake'")) {
+ assertTrue(rows.next());
+ assertEquals(0, rows.getInt(1));
+ }
+ statement.execute("DROP SCHEMA IF EXISTS lake");
+ statement.execute("CREATE EXTERNAL SCHEMA lake FROM DATA CATALOG DATABASE '" + glueDatabase
+ + "' IAM_ROLE '" + ROLE_ARN + "'");
+ assertTotal(statement, 30);
+ lifecycleDataStatement("DROP TABLE lake.events");
+ glueService.deleteDatabase(glueDatabase);
+ glueDatabase = null;
+ statement.execute("DROP SCHEMA lake");
+ }
+ }
+
+ private static void assertTotal(Statement statement, long total) throws SQLException {
+ try (ResultSet rows = statement.executeQuery("SELECT COALESCE(SUM(amount),0) FROM lake.events")) {
+ assertTrue(rows.next());
+ assertEquals(total, rows.getLong(1));
+ }
+ }
+
+ private void lifecycleDataStatement(String sql) {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ String queryId = RestAssuredJsonUtils.awsAction("RedshiftData", "ExecuteStatement", """
+ {"ClusterIdentifier":"%s","Database":"dev","DbUser":"admin","Sql":"%s","WaitTimeSeconds":30}
+ """.formatted(clusterId, sql.replace("\\", "\\\\").replace("\"", "\\\"")))
+ .then().statusCode(200).extract().path("Id");
+ assertEquals("FINISHED", RestAssuredJsonUtils.awsAction("RedshiftData", "DescribeStatement",
+ "{\"Id\":\"" + queryId + "\"}").then().statusCode(200).extract().path("Status"));
+ }
+
private static Connection waitForConnection(Cluster cluster, String username, String password) throws SQLException {
String url = "jdbc:postgresql://127.0.0.1:" + cluster.getEndpoint().getPort() + "/dev";
return Awaitility.await().atMost(Duration.ofSeconds(30)).pollDelay(Duration.ZERO)
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
index ca5f687c62..e460830b19 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/proxy/RedshiftInterceptingBridgeTest.java
@@ -2,6 +2,8 @@ package io.github.hectorvent.floci.services.redshift.proxy;
import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumInterceptor;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumQueryPreparation;
+import io.github.hectorvent.floci.services.redshift.spectrum.SpectrumSession;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.jboss.logging.Logger;
@@ -355,6 +357,32 @@ class RedshiftInterceptingBridgeTest {
assertEquals('T', fromBridge.read());
}
+ @Test
+ @Timeout(20)
+ void parsingAndDescribingExternalDdlDoesNotExecuteCatalogChanges() throws Exception {
+ String sql = "ALTER TABLE lake.events ADD PARTITION (dt='today') LOCATION 's3://bucket/today/'";
+ SpectrumInterceptor interceptor = inlineMock(SpectrumInterceptor.class);
+ SpectrumQueryPreparation preparation = Mockito.mock(SpectrumQueryPreparation.class);
+ Mockito.when(interceptor.preparation()).thenReturn(preparation);
+ Mockito.when(preparation.referencesExternal(Mockito.eq(sql), Mockito.any())).thenReturn(true);
+ Mockito.when(interceptor.plan(Mockito.eq(sql), Mockito.any())).thenReturn(new SpectrumInterceptor.Plan.ExternalDdl(
+ sql, new SpectrumSession("000000000000", "000000000000:c", "dev", List.of(), false)));
+ startBridge(interceptor);
+ PostgresWireDecoder.ParseMessage parse = new PostgresWireDecoder.ParseMessage("ddl", sql, List.of());
+ testClientEnd.getOutputStream().write(PostgresWireDecoder.encodeParse(parse, sql));
+ testClientEnd.getOutputStream().write(frame('D', concat(new byte[]{'S'}, cString("ddl"))));
+ testClientEnd.getOutputStream().flush();
+ InputStream response = testClientEnd.getInputStream();
+ assertEquals('1', response.read());
+ skipFrameBody(response);
+ assertEquals('t', response.read());
+ skipFrameBody(response);
+ assertEquals('n', response.read());
+ skipFrameBody(response);
+ Mockito.verify(preparation, Mockito.never()).prepare(Mockito.anyString(), Mockito.any(), Mockito.any());
+ Mockito.verify(interceptor, Mockito.never()).execute(Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any());
+ }
+
private static void skipFrameBody(InputStream in) throws IOException {
int length = ByteBuffer.wrap(in.readNBytes(4)).getInt();
in.readNBytes(length - 4);
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalLifecycleTest.java
new file mode 100644
index 0000000000..6d06ee6aec
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalLifecycleTest.java
@@ -0,0 +1,383 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static io.github.hectorvent.floci.testing.ThreadStates.awaitBlockedOrDone;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+class ExternalLifecycleTest {
+ private static final SpectrumSession SESSION = new SpectrumSession("000000000000", "000000000000:c",
+ "dev", List.of(), false);
+ private static final ExternalSchemaBinding BINDING = new ExternalSchemaBinding(SESSION.accountId(),
+ SESSION.clusterKey(), "dev", "lake", "analytics", "arn:aws:iam::000000000000:role/R");
+ private static final String ADD = "ALTER TABLE lake.sales ADD PARTITION (dt='2026-10-17') LOCATION 's3://bucket/day/'";
+ private final ExternalCatalogRegistry registry = mock(ExternalCatalogRegistry.class);
+ private final ExternalTableMaterializer materializer = mock(ExternalTableMaterializer.class);
+ private final GlueService glue = mock(GlueService.class);
+ private final BackendSql backend = mock(BackendSql.class);
+ private final ExternalMetadataWriter metadata = spy(new ExternalMetadataWriter(glue, new ObjectMapper()));
+ private final ExternalCatalogLocks catalogLocks = new ExternalCatalogLocks();
+ private final SpectrumQueryPreparation preparation = new SpectrumQueryPreparation(registry, materializer,
+ new ExternalStatementParser(), glue, metadata, catalogLocks);
+ private Table table;
+
+ @BeforeEach
+ void setUp() {
+ when(registry.list(SESSION.accountId(), SESSION.clusterKey(), "dev")).thenReturn(List.of(BINDING));
+ Column key = new Column();
+ key.setName("dt");
+ key.setType("string");
+ table = new Table();
+ table.setName("sales");
+ table.setPartitionKeys(List.of(key));
+ table.setStorageDescriptor(new StorageDescriptor());
+ when(glue.getTable("analytics", "sales")).thenReturn(table);
+ doNothing().when(metadata).refresh(any(), anyString(), any());
+ }
+
+ @Test
+ void registersPartitionAndInvalidatesMaterializedRows() {
+ assertTrue(preparation.handlesDdl(ADD, SESSION));
+ assertTrue(preparation.prepare(ADD, SESSION, backend));
+ ArgumentCaptor<Partition> captured = ArgumentCaptor.forClass(Partition.class);
+ verify(glue).createPartition(eq("analytics"), eq("sales"), captured.capture());
+ assertEquals(List.of("2026-10-17"), captured.getValue().getValues());
+ assertEquals("s3://bucket/day/", captured.getValue().getStorageDescriptor().getLocation());
+ verify(materializer).invalidateTable(SESSION.clusterKey(), "dev", "lake", "sales");
+ verify(metadata).refresh(backend, SESSION.accountId(), BINDING);
+ }
+
+ @Test
+ void duplicatePartitionIsAnErrorAndIfNotExistsPreservesItsLocation() {
+ Partition existing = new Partition();
+ existing.setValues(List.of("2026-10-17"));
+ when(glue.getPartitions("analytics", "sales")).thenReturn(List.of(existing));
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare(ADD, SESSION, backend));
+ assertEquals("42710", error.sqlState());
+ assertTrue(preparation.prepare(ADD.replace("ADD PARTITION", "ADD IF NOT EXISTS PARTITION"), SESSION, backend));
+ verify(glue, never()).createPartition(anyString(), anyString(), any());
+ }
+
+ @Test
+ void validatesEveryPartitionBeforeWritingAny() {
+ String sql = ADD + " PARTITION (wrong='x') LOCATION 's3://bucket/invalid/'";
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare(sql, SESSION, backend));
+ assertEquals("22023", error.sqlState());
+ verify(glue, never()).createPartition(anyString(), anyString(), any());
+ }
+
+ @Test
+ void duplicateWithinTheSameRequestDoesNotPartiallyRegister() {
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare(ADD + " PARTITION (dt='2026-10-17') LOCATION 's3://bucket/other/'", SESSION, backend));
+ assertEquals("42710", error.sqlState());
+ verify(glue, never()).createPartition(anyString(), anyString(), any());
+ }
+
+ @Test
+ void dropTableRemovesGlueDefinitionAndColdOrCachedBackendRelation() {
+ assertTrue(preparation.handlesDdl("DROP TABLE lake.sales", SESSION));
+ assertTrue(preparation.prepare("DROP TABLE lake.sales", SESSION, backend));
+ verify(backend).execute("DROP TABLE IF EXISTS \"lake\".\"sales\" RESTRICT");
+ verify(glue).deleteTable("analytics", "sales");
+ verify(materializer).forget(SESSION.clusterKey(), "dev", "lake", "sales");
+ verify(metadata).refresh(backend, SESSION.accountId(), BINDING);
+ verify(materializer, never()).ensureCurrent(any(), any(), any(), anyString());
+ }
+
+ @Test
+ void failedDependencyCheckKeepsGlueTable() {
+ doThrow(new SpectrumReadException("2BP01", "dependent view")).when(backend).execute(anyString());
+ assertThrows(SpectrumReadException.class, () -> preparation.prepare("DROP TABLE lake.sales", SESSION, backend));
+ verify(glue, never()).deleteTable(anyString(), anyString());
+ verifyNoInteractions(materializer);
+ }
+
+ @Test
+ void dropMissingTableRequiresIfExistsAndCleansStaleCache() {
+ when(glue.getTable("analytics", "sales"))
+ .thenThrow(new AwsException("EntityNotFoundException", "gone", 400));
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare("DROP TABLE lake.sales", SESSION, backend));
+ assertEquals("42P01", error.sqlState());
+ verifyNoInteractions(backend);
+ assertTrue(preparation.prepare("DROP TABLE IF EXISTS lake.sales CASCADE", SESSION, backend));
+ verify(backend).execute("DROP TABLE IF EXISTS \"lake\".\"sales\" CASCADE");
+ verify(materializer).forget(SESSION.clusterKey(), "dev", "lake", "sales");
+ verify(glue, never()).deleteTable(anyString(), anyString());
+ }
+
+ @Test
+ void restrictChecksGlueTablesThatHaveNeverBeenLoaded() {
+ when(glue.getTables("analytics")).thenReturn(List.of(table));
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare("DROP SCHEMA lake", SESSION, backend));
+ assertEquals("2BP01", error.sqlState());
+ verify(backend, never()).execute(startsWith("DROP SCHEMA"));
+ verify(registry, never()).unbind(anyString(), anyString(), anyString(), anyString());
+ }
+
+ @Test
+ void cascadeUnregistersSchemaWithoutDeletingGlueDatabaseOrTables() {
+ assertTrue(preparation.handlesDdl("DROP SCHEMA lake CASCADE", SESSION));
+ assertTrue(preparation.prepare("DROP SCHEMA lake CASCADE", SESSION, backend));
+ verify(backend).execute("DROP SCHEMA \"lake\" CASCADE; SELECT floci_internal.purge_external_schema('lake')");
+ verify(registry).unbind(SESSION.accountId(), SESSION.clusterKey(), "dev", "lake");
+ verify(materializer).forgetSchema(SESSION.clusterKey(), "dev", "lake");
+ verify(glue, never()).deleteDatabase(anyString());
+ verify(glue, never()).deleteTable(anyString(), anyString());
+ }
+
+ @Test
+ void failedSchemaDropRetainsBindingAndCache() {
+ doThrow(new SpectrumReadException("42501", "not owner")).when(backend).execute(anyString());
+ assertThrows(SpectrumReadException.class, () -> preparation.prepare("DROP SCHEMA lake CASCADE", SESSION, backend));
+ verify(registry, never()).unbind(anyString(), anyString(), anyString(), anyString());
+ verifyNoInteractions(materializer);
+ }
+
+ @Test
+ void rejectsLifecycleInsideTransactionsBeforeMutatingStores() {
+ SpectrumSession transaction = new SpectrumSession(SESSION.accountId(), SESSION.clusterKey(), "dev", List.of(), true);
+ for (String sql : List.of(ADD, "DROP TABLE lake.sales", "DROP SCHEMA lake CASCADE")) {
+ assertTrue(preparation.handlesDdl(sql, transaction));
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare(sql, transaction, backend));
+ assertEquals("25001", error.sqlState());
+ }
+ verifyNoInteractions(glue, backend, materializer);
+ }
+
+ @Test
+ void leavesInternalLifecycleToPostgres() {
+ for (String sql : List.of("DROP TABLE public.sales", "DROP SCHEMA public CASCADE",
+ "ALTER TABLE public.sales ADD COLUMN total INTEGER")) {
+ assertFalse(preparation.handlesDdl(sql, SESSION));
+ assertFalse(preparation.prepare(sql, SESSION, backend));
+ }
+ verifyNoInteractions(glue, backend, materializer);
+ }
+
+ @Test
+ void malformedOrRepeatedPartitionKeysNeverReachGlue() {
+ for (String keys : List.of("dt='x', garbage", "dt='x', dt='y'", "dt='x' garbage")) {
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () -> preparation.prepare(
+ "ALTER TABLE lake.sales ADD PARTITION (" + keys + ") LOCATION 's3://bucket/day/'", SESSION, backend));
+ assertEquals("42601", error.sqlState());
+ }
+ verifyNoInteractions(glue, backend, materializer);
+ }
+
+ @Test
+ void ifExistsDoesNotSuppressAccessDenied() {
+ when(glue.getTable("analytics", "sales"))
+ .thenThrow(new AwsException("AccessDeniedException", "denied", 403));
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare("DROP TABLE IF EXISTS lake.sales", SESSION, backend));
+ assertEquals("42501", error.sqlState());
+ verifyNoInteractions(backend, materializer);
+ }
+
+ @Test
+ void quotesSchemaAndTableNamesBeforeSendingBackendDdl() {
+ ExternalSchemaBinding quoted = new ExternalSchemaBinding(SESSION.accountId(), SESSION.clusterKey(),
+ "dev", "a\"b", "analytics", BINDING.iamRoleArn());
+ when(registry.list(SESSION.accountId(), SESSION.clusterKey(), "dev")).thenReturn(List.of(quoted));
+ assertTrue(preparation.prepare("DROP TABLE IF EXISTS \"a\"\"b\".\"sales\"", SESSION, backend));
+ verify(backend).execute("DROP TABLE IF EXISTS \"a\"\"b\".\"sales\" RESTRICT");
+ }
+
+ @Test
+ void backendAuthorizationFailsBeforeAnyCatalogMutation() {
+ doThrow(new SpectrumReadException("42501", "not owner")).when(backend).execute(anyString());
+ for (String sql : List.of(ADD, "DROP TABLE lake.sales")) {
+ SpectrumReadException error = assertThrows(SpectrumReadException.class,
+ () -> preparation.prepare(sql, SESSION, backend));
+ assertEquals("42501", error.sqlState());
+ }
+ verify(glue, never()).createPartition(anyString(), anyString(), any());
+ verify(glue, never()).deleteTable(anyString(), anyString());
+ }
+
+ @Test
+ void rejectsCaseVariantsOfTheSamePartitionKey() {
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () -> preparation.prepare(
+ "ALTER TABLE lake.sales ADD PARTITION (\"DT\"='upper', dt='lower') LOCATION 's3://bucket/day/'",
+ SESSION, backend));
+ assertEquals("42601", error.sqlState());
+ verifyNoInteractions(glue, backend, materializer);
+ }
+
+ @Test
+ void checksSchemaOwnershipBeforeReadingGlueTables() {
+ doThrow(new SpectrumReadException("42501", "not owner")).when(backend).execute(anyString());
+ SpectrumReadException error = assertThrows(SpectrumReadException.class,
+ () -> preparation.prepare("DROP SCHEMA lake RESTRICT", SESSION, backend));
+ assertEquals("42501", error.sqlState());
+ verifyNoInteractions(glue);
+ verify(registry, never()).unbind(anyString(), anyString(), anyString(), anyString());
+ }
+
+ @Test
+ void dropsLocalSchemaWhenGlueDatabaseIsAlreadyGone() {
+ when(glue.getTables("analytics")).thenThrow(new AwsException("EntityNotFoundException", "gone", 400));
+ assertTrue(preparation.prepare("DROP SCHEMA IF EXISTS lake", SESSION, backend));
+ verify(backend).execute("DROP SCHEMA IF EXISTS \"lake\" RESTRICT; SELECT floci_internal.purge_external_schema('lake')");
+ verify(registry).unbind(SESSION.accountId(), SESSION.clusterKey(), "dev", "lake");
+ verify(materializer).forgetSchema(SESSION.clusterKey(), "dev", "lake");
+ }
+
+ @Test
+ void schemaDropDoesNotSuppressOtherGlueFailures() {
+ when(glue.getTables("analytics")).thenThrow(new AwsException("AccessDeniedException", "denied", 403));
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare("DROP SCHEMA IF EXISTS lake", SESSION, backend));
+ assertEquals("42501", error.sqlState());
+ verify(backend, never()).execute(startsWith("DROP SCHEMA"));
+ verify(registry, never()).unbind(anyString(), anyString(), anyString(), anyString());
+ }
+
+ @Test
+ void rollsBackOnlyNewPartitionsWhenLaterWriteFails() {
+ Partition existing = new Partition();
+ existing.setValues(List.of("existing"));
+ when(glue.getPartitions("analytics", "sales")).thenReturn(List.of(existing));
+ doAnswer(invocation -> {
+ Partition partition = invocation.getArgument(2);
+ if (partition.getValues().equals(List.of("fail"))) {
+ throw new AwsException("InternalServiceException", "write failed", 500);
+ }
+ return null;
+ }).when(glue).createPartition(anyString(), anyString(), any());
+ String sql = ADD.replace("ADD PARTITION", "ADD IF NOT EXISTS PARTITION")
+ + " PARTITION (dt='existing') LOCATION 's3://bucket/old/'"
+ + " PARTITION (dt='fail') LOCATION 's3://bucket/fail/'";
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> preparation.prepare(sql, SESSION, backend));
+ assertEquals("write failed", error.getMessage());
+ verify(glue).deletePartition("analytics", "sales", List.of("2026-10-17"));
+ verify(glue, never()).deletePartition("analytics", "sales", List.of("existing"));
+ verify(glue, never()).deletePartition("analytics", "sales", List.of("fail"));
+ }
+
+ @Test
+ void rollsBackPartitionsWhenMetadataRefreshFailsWithoutMaskingTheFailure() {
+ SpectrumReadException failure = new SpectrumReadException("58030", "refresh failed");
+ doThrow(failure).when(metadata).refresh(backend, SESSION.accountId(), BINDING);
+ doThrow(new AwsException("InternalServiceException", "cleanup failed", 500)).when(glue)
+ .deletePartition("analytics", "sales", List.of("2026-10-17"));
+ assertSame(failure, assertThrows(SpectrumReadException.class, () -> preparation.prepare(ADD, SESSION, backend)));
+ verify(glue).deletePartition("analytics", "sales", List.of("2026-10-17"));
+ assertEquals(1, failure.getSuppressed().length);
+ verify(materializer).invalidateTable(SESSION.clusterKey(), "dev", "lake", "sales");
+ }
+
+ @Test
+ void implicitExtendedTransactionTimesOutWhileCatalogMutationIsWaiting() throws Exception {
+ when(backend.permitsCachePublication()).thenReturn(false);
+ ExternalCatalogLocks shortLocks = new ExternalCatalogLocks(Duration.ofMillis(20));
+ SpectrumQueryPreparation timedPreparation = new SpectrumQueryPreparation(registry, materializer,
+ new ExternalStatementParser(), glue, metadata, shortLocks);
+ try (ExecutorService executor = Executors.newSingleThreadExecutor()) {
+ ExternalCatalogLocks.Guard mutation = shortLocks.write(BINDING);
+ try {
+ Future<Boolean> read = executor.submit(
+ () -> timedPreparation.prepare("SELECT * FROM lake.sales", SESSION, backend));
+ ExecutionException failure = assertThrows(ExecutionException.class,
+ () -> read.get(5, TimeUnit.SECONDS));
+ SpectrumReadException timeout = assertInstanceOf(SpectrumReadException.class, failure.getCause());
+ assertEquals("58030", timeout.sqlState());
+ verifyNoInteractions(materializer, glue);
+ } finally {
+ mutation.close();
+ }
+ }
+ }
+
+ @Test
+ void removesAllNewPartitionsWhenMetadataRefreshFails() {
+ SpectrumReadException failure = new SpectrumReadException("58030", "refresh failed");
+ doThrow(failure).when(metadata).refresh(backend, SESSION.accountId(), BINDING);
+ String sql = ADD + " PARTITION (dt='2026-10-18') LOCATION 's3://bucket/next/'";
+
+ assertSame(failure, assertThrows(SpectrumReadException.class,
+ () -> preparation.prepare(sql, SESSION, backend)));
+
+ verify(glue).deletePartition("analytics", "sales", List.of("2026-10-17"));
+ verify(glue).deletePartition("analytics", "sales", List.of("2026-10-18"));
+ verify(materializer).invalidateTable(SESSION.clusterKey(), "dev", "lake", "sales");
+ }
+
+ @Test
+ void concurrentIfNotExistsAddsShareTheGlueTableLock() throws Exception {
+ List<Partition> stored = new CopyOnWriteArrayList<>();
+ CountDownLatch checking = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicInteger checks = new AtomicInteger();
+ when(glue.getTable("ANALYTICS", "sales")).thenReturn(table);
+ when(glue.getPartitions(anyString(), eq("sales"))).thenAnswer(invocation -> {
+ List<Partition> snapshot = List.copyOf(stored);
+ if (checks.incrementAndGet() == 1) {
+ checking.countDown();
+ assertTrue(release.await(10, TimeUnit.SECONDS));
+ }
+ return snapshot;
+ });
+ doAnswer(invocation -> {
+ stored.add(invocation.getArgument(2));
+ return null;
+ }).when(glue).createPartition(anyString(), anyString(), any());
+ String sql = ADD.replace("ADD PARTITION", "ADD IF NOT EXISTS PARTITION");
+ SpectrumSession otherSession = new SpectrumSession(SESSION.accountId(), SESSION.accountId() + ":other",
+ "dev", List.of(), false);
+ ExternalSchemaBinding alias = new ExternalSchemaBinding(SESSION.accountId(), otherSession.clusterKey(),
+ "dev", "alias", "ANALYTICS", BINDING.iamRoleArn());
+ when(registry.list(SESSION.accountId(), otherSession.clusterKey(), "dev")).thenReturn(List.of(alias));
+ SpectrumQueryPreparation otherPreparation = new SpectrumQueryPreparation(registry, materializer,
+ new ExternalStatementParser(), glue, metadata, catalogLocks);
+ try (ExecutorService executor = Executors.newFixedThreadPool(2)) {
+ Future<Boolean> first = executor.submit(() -> preparation.prepare(sql, SESSION, backend));
+ try {
+ assertTrue(checking.await(10, TimeUnit.SECONDS));
+ AtomicReference<Thread> secondThread = new AtomicReference<>();
+ Future<Boolean> second = executor.submit(() -> {
+ secondThread.set(Thread.currentThread());
+ return otherPreparation.prepare(
+ sql.replace("lake.sales", "alias.sales"), otherSession, backend);
+ });
+ awaitBlockedOrDone(secondThread, second);
+ assertFalse(second.isDone());
+ release.countDown();
+ assertTrue(first.get(10, TimeUnit.SECONDS));
+ assertTrue(second.get(10, TimeUnit.SECONDS));
+ assertEquals(1, stored.size());
+ } finally {
+ release.countDown();
+ }
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java
index 57fddd57ad..27928c4203 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalTableMaterializerTest.java
@@ -32,14 +32,18 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import static io.github.hectorvent.floci.testing.ThreadStates.awaitBlockedOrDone;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
@@ -452,6 +456,20 @@ class ExternalTableMaterializerTest {
assertThat(backend.statements.getLast(), containsString("CREATE TABLE \"analytics\".\"events\""));
}
+ @Test
+ void invalidatingDataKeepsTheExistingTableAndItsDependencies() {
+ when(glue.getTable("lake", "events")).thenReturn(csvTable());
+ materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ materializer.invalidateTable(BINDING.clusterKey(), "dev", "analytics", "events");
+ backend.statements.clear();
+
+ assertThat(materializer.ensureCurrent(backend, session(false), BINDING, "events"),
+ equalTo(ExternalTableMaterializer.Outcome.LOADED));
+ String swap = backend.statements.getLast();
+ assertThat(swap, containsString("TRUNCATE \"analytics\".\"events\""));
+ assertThat(swap, not(containsString("DROP TABLE IF EXISTS \"analytics\".\"events\"")));
+ }
+
private void forceReload() {
when(s3.listObjectsWithPrefixes(eq("bucket"), eq("events/"), eq(""), eq(1000), any(), any()))
.thenReturn(new S3Service.ListObjectsResult(
@@ -766,9 +784,13 @@ class ExternalTableMaterializerTest {
Future<ExternalTableMaterializer.Outcome> first = executor.submit(
() -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
assertThat(loading.await(10, TimeUnit.SECONDS), equalTo(true));
- Future<ExternalTableMaterializer.Outcome> second = executor.submit(
- () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
- assertThrows(TimeoutException.class, () -> second.get(300, TimeUnit.MILLISECONDS));
+ AtomicReference<Thread> secondThread = new AtomicReference<>();
+ Future<ExternalTableMaterializer.Outcome> second = executor.submit(() -> {
+ secondThread.set(Thread.currentThread());
+ return materializer.ensureCurrent(backend, session(false), BINDING, "events");
+ });
+ awaitBlockedOrDone(secondThread, second);
+ assertFalse(second.isDone());
release.countDown();
@@ -780,6 +802,56 @@ class ExternalTableMaterializerTest {
}
}
+ @Test
+ void dropWaitsForAnEarlierLoadBeforeDeletingTheMaterializedTable() throws Exception {
+ AtomicReference<Table> stored = new AtomicReference<>(csvTable());
+ AtomicInteger lookups = new AtomicInteger();
+ CountDownLatch reading = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ when(glue.getTable("lake", "events")).thenAnswer(invocation -> {
+ Table snapshot = stored.get();
+ if (lookups.incrementAndGet() == 1) {
+ reading.countDown();
+ assertTrue(release.await(10, TimeUnit.SECONDS));
+ }
+ if (snapshot == null) {
+ throw new AwsException("EntityNotFoundException", "gone", 400);
+ }
+ return snapshot;
+ });
+ doAnswer(invocation -> {
+ stored.set(null);
+ return null;
+ }).when(glue).deleteTable("lake", "events");
+ ExternalCatalogRegistry registry = mock(ExternalCatalogRegistry.class);
+ when(registry.list(ACCOUNT, BINDING.clusterKey(), "dev")).thenReturn(List.of(BINDING));
+ SpectrumQueryPreparation preparation = new SpectrumQueryPreparation(registry, materializer,
+ new ExternalStatementParser(), glue, mock(ExternalMetadataWriter.class), materializer.catalogLocks());
+ try (ExecutorService executor = Executors.newFixedThreadPool(2)) {
+ Future<ExternalTableMaterializer.Outcome> load = executor.submit(
+ () -> materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+ try {
+ assertTrue(reading.await(10, TimeUnit.SECONDS));
+ AtomicReference<Thread> dropThread = new AtomicReference<>();
+ Future<Boolean> drop = executor.submit(() -> {
+ dropThread.set(Thread.currentThread());
+ return preparation.prepare("DROP TABLE analytics.events", session(false), backend);
+ });
+ awaitBlockedOrDone(dropThread, drop);
+ assertFalse(drop.isDone());
+ release.countDown();
+ assertEquals(ExternalTableMaterializer.Outcome.LOADED, load.get(10, TimeUnit.SECONDS));
+ assertTrue(drop.get(10, TimeUnit.SECONDS));
+ assertEquals("DROP TABLE IF EXISTS \"analytics\".\"events\" RESTRICT", backend.statements.getLast());
+ assertEquals(ExternalTableMaterializer.Outcome.NOT_EXTERNAL,
+ materializer.ensureCurrent(backend, session(false), BINDING, "events"));
+ assertTrue(materializer.loadedTables(BINDING.clusterKey(), "dev", "analytics").isEmpty());
+ } finally {
+ release.countDown();
+ }
+ }
+ }
+
private Partition partitionAt(String inputFormat, Table table) {
StorageDescriptor descriptor = new StorageDescriptor();
descriptor.setLocation("s3://bucket/events/");
diff --git a/src/test/java/io/github/hectorvent/floci/testing/ThreadStates.java b/src/test/java/io/github/hectorvent/floci/testing/ThreadStates.java
index 8a632d9a03..41afad566b 100644
--- a/src/test/java/io/github/hectorvent/floci/testing/ThreadStates.java
+++ b/src/test/java/io/github/hectorvent/floci/testing/ThreadStates.java
@@ -14,8 +14,8 @@ public final class ThreadStates {
private ThreadStates() {}
/**
- * Returns once {@code thread} is blocked entering a monitor, or {@code task} has finished,
- * whichever comes first. Code that serializes correctly leaves the second request blocked
+ * Returns once {@code thread} is blocked entering a monitor or parked on a lock, or {@code task} has
+ * finished, whichever comes first. Code that serializes correctly leaves the second request blocked
* behind the first; code that does not lets it run to the end, which the caller's assertions
* then catch. Polls every 5 ms and gives up after five seconds rather than hang a broken test.
*/
@@ -25,9 +25,9 @@ public final class ThreadStates {
}
/**
- * Returns once {@code thread} is blocked entering a monitor or is no longer alive. Same wait as
- * the {@link Future} overload, for a test that owns the {@link Thread} directly; gives up after
- * ten seconds and fails, since a thread still running unblocked by then is not overlapping the
+ * Returns once {@code thread} is blocked entering a monitor, parked on a lock, or is no longer alive.
+ * Same wait as the {@link Future} overload, for a test that owns the {@link Thread} directly; gives up
+ * after ten seconds and fails, since a thread still running unblocked by then is not overlapping the
* way the test needs.
*/
public static void awaitBlockedOrDone(Thread thread) throws InterruptedException {
@@ -48,6 +48,10 @@ public final class ThreadStates {
}
private static boolean isBlocked(Thread thread) {
- return thread != null && thread.getState() == Thread.State.BLOCKED;
+ if (thread == null) {
+ return false;
+ }
+ Thread.State state = thread.getState();
+ return state == Thread.State.BLOCKED || state == Thread.State.WAITING;
}
}