Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(redshift): add external catalog statement parsing and metadata writers (#4679)

Vũ Thế Huyd7d2f73e99c4
diff --git a/docs/services/redshift.md b/docs/services/redshift.md
index 0092c1b78b..b1508e23c3 100644
--- a/docs/services/redshift.md
+++ b/docs/services/redshift.md
@@ -251,7 +251,8 @@ CREATE EXTERNAL TABLE <schema>.<table> (<name> <type>, ...)
```
Supported column types are `VARCHAR`, `CHAR`, `INTEGER`, `BIGINT`, `DECIMAL`, `BOOLEAN`, `DATE`,
-and `TIMESTAMP`. CSV reads support `DELIMITER`, `QUOTE`, `ESCAPE`, `NULL AS`, and
+and `TIMESTAMP`. `CHAR` defaults to length 1 and `VARCHAR` defaults to length 256 when no length
+is specified. CSV reads support `DELIMITER`, `QUOTE`, `ESCAPE`, `NULL AS`, and
`skip.header.line.count`. Objects under the location are read in lexicographic key order.
The query path supports `SELECT *`, explicit column projections, and simple `WHERE` predicates
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java
new file mode 100644
index 0000000000..11e541fc26
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java
@@ -0,0 +1,8 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import java.io.InputStream;
+
+public interface BackendSql {
+ void execute(String sql);
+ long copyIn(String copySql, InputStream data);
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistry.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistry.java
new file mode 100644
index 0000000000..7636b045dc
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistry.java
@@ -0,0 +1,80 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.core.common.Resettable;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import jakarta.enterprise.context.ApplicationScoped;
+
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+
+@ApplicationScoped
+public class ExternalCatalogRegistry implements Resettable {
+ private static final char SEPARATOR = '|';
+ private final AccountAwareStorageBackend<ExternalSchemaBinding> bindings;
+
+ public ExternalCatalogRegistry(StorageFactory storageFactory) {
+ bindings = storageFactory.create("redshift", "redshift-external-schemas.json", new TypeReference<>() { });
+ }
+
+ public synchronized void bind(ExternalSchemaBinding binding) {
+ String key = key(binding.clusterKey(), binding.databaseName(), binding.schemaName());
+ if (bindings.getForAccount(binding.accountId(), key).isPresent()) {
+ throw new SpectrumSqlException("42P06", "schema \"" + binding.schemaName() + "\" already exists");
+ }
+ bindings.putForAccount(binding.accountId(), key, binding);
+ bindings.flush();
+ }
+
+ public Optional<ExternalSchemaBinding> find(String accountId, String clusterKey, String databaseName, String schemaName) {
+ return bindings.getForAccount(accountId, key(clusterKey, databaseName, schemaName));
+ }
+
+ public List<ExternalSchemaBinding> list(String accountId, String clusterKey, String databaseName) {
+ String prefix = prefix(clusterKey, databaseName);
+ return bindings.scanForAccount(accountId, key -> key.startsWith(prefix)).stream()
+ .sorted(Comparator.comparing(ExternalSchemaBinding::schemaName)).toList();
+ }
+
+ public synchronized void unbind(String accountId, String clusterKey, String databaseName, String schemaName) {
+ bindings.deleteForAccount(accountId, key(clusterKey, databaseName, schemaName));
+ bindings.flush();
+ }
+
+ public synchronized void removeCluster(String accountId, String clusterKey) {
+ String prefix = prefix(clusterKey);
+ for (String key : bindings.keysForAccount(accountId)) {
+ if (key.startsWith(prefix)) {
+ bindings.deleteForAccount(accountId, key);
+ }
+ }
+ bindings.flush();
+ }
+
+ @Override
+ public synchronized void clear() {
+ bindings.clear();
+ bindings.flush();
+ }
+
+ private static String key(String clusterKey, String databaseName, String schemaName) {
+ return prefix(clusterKey, databaseName) + encode(schemaName);
+ }
+
+ private static String prefix(String clusterKey, String databaseName) {
+ return prefix(clusterKey) + encode(databaseName) + SEPARATOR;
+ }
+
+ private static String prefix(String clusterKey) {
+ return encode(clusterKey) + SEPARATOR;
+ }
+
+ /** Percent-encodes a component so a quoted identifier containing the separator cannot collide with another key. */
+ private static String encode(String component) {
+ return URLEncoder.encode(component, StandardCharsets.UTF_8);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriter.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriter.java
new file mode 100644
index 0000000000..fb9c1dd727
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriter.java
@@ -0,0 +1,189 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import jakarta.enterprise.context.ApplicationScoped;
+import org.jboss.logging.Logger;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Mirrors Glue metadata into the tables exposed by the SVV external catalog views. */
+@ApplicationScoped
+public class ExternalMetadataWriter {
+ private static final Logger LOG = Logger.getLogger(ExternalMetadataWriter.class);
+ private final GlueService glueService;
+ private final ObjectMapper objectMapper;
+
+ public ExternalMetadataWriter(GlueService glueService, ObjectMapper objectMapper) {
+ this.glueService = glueService;
+ this.objectMapper = objectMapper;
+ }
+
+ public void refresh(BackendSql backend, String accountId, ExternalSchemaBinding binding) {
+ List<Table> tables = RequestScopes.callAs(accountId, () -> glueService.getTables(binding.glueDatabase()));
+ Map<String, List<Partition>> partitions = new LinkedHashMap<>();
+ for (Table table : tables) {
+ partitions.put(table.getName(), RequestScopes.callAs(accountId,
+ () -> glueService.getPartitions(binding.glueDatabase(), table.getName())));
+ }
+ run(backend, refreshSql(binding, tables, partitions), false);
+ }
+
+ public void purge(BackendSql backend, String schemaName) {
+ run(backend, purgeSql(schemaName), true);
+ }
+
+ private void run(BackendSql backend, String sql, boolean tolerateMissingBootstrapTables) {
+ try {
+ backend.execute(sql);
+ } catch (SpectrumReadException exception) {
+ if (!tolerateMissingBootstrapTables || !"42P01".equals(exception.sqlState())) {
+ throw exception;
+ }
+ LOG.warnv("Spectrum metadata tables are missing; restart the cluster to re-run the catalog bootstrap: {0}",
+ exception.getMessage());
+ }
+ }
+
+ String purgeSql(String schemaName) {
+ return "SELECT floci_internal.purge_external_schema(" + literal(schemaName) + ")";
+ }
+
+ String refreshSql(ExternalSchemaBinding binding, List<Table> tables,
+ Map<String, List<Partition>> partitionsByTable) {
+ List<Map<String, Object>> tableRows = new ArrayList<>();
+ List<Map<String, Object>> columnRows = new ArrayList<>();
+ List<Map<String, Object>> partitionRows = new ArrayList<>();
+ for (Table table : tables) {
+ StorageDescriptor descriptor = table.getStorageDescriptor();
+ String serdeLibrary = serdeLibrary(descriptor);
+ String serdeParameters = json(serdeParameters(descriptor));
+ int compressed = descriptor != null && Boolean.TRUE.equals(descriptor.getCompressed()) ? 1 : 0;
+ tableRows.add(row(
+ "tablename", table.getName(),
+ "tabletype", "VIRTUAL_VIEW".equals(table.getTableType()) ? "VIEW" : "TABLE",
+ "location", descriptor == null ? "" : descriptor.getLocation(),
+ "input_format", descriptor == null ? "" : descriptor.getInputFormat(),
+ "output_format", descriptor == null ? "" : descriptor.getOutputFormat(),
+ "serialization_lib", serdeLibrary,
+ "serde_parameters", serdeParameters,
+ "compressed", compressed,
+ "parameters", json(table.getParameters())));
+ appendColumns(columnRows, table);
+ List<Partition> partitions = partitionsByTable.get(table.getName());
+ if (partitions != null) {
+ for (Partition partition : partitions) {
+ appendPartition(partitionRows, table.getName(), partition, descriptor);
+ }
+ }
+ }
+ Map<String, Object> payload = row(
+ "databasename", binding.glueDatabase(),
+ "esoptions", json(Map.of("IAM_ROLE", binding.iamRoleArn())),
+ "tables", tableRows,
+ "columns", columnRows,
+ "partitions", partitionRows);
+ return "SELECT floci_internal.refresh_external_catalog(" + literal(binding.schemaName()) + ", "
+ + literal(json(payload)) + "::jsonb)";
+ }
+
+ private static void appendColumns(List<Map<String, Object>> rows, Table table) {
+ StorageDescriptor descriptor = table.getStorageDescriptor();
+ List<Column> dataColumns = descriptor == null || descriptor.getColumns() == null
+ ? List.of() : descriptor.getColumns();
+ List<Column> partitionKeys = table.getPartitionKeys() == null ? List.of() : table.getPartitionKeys();
+ int number = 1;
+ for (Column column : dataColumns) {
+ appendColumn(rows, table.getName(), column, number++, 0);
+ }
+ int partitionOrder = 1;
+ for (Column column : partitionKeys) {
+ appendColumn(rows, table.getName(), column, number++, partitionOrder++);
+ }
+ }
+
+ private static void appendColumn(List<Map<String, Object>> rows, String tableName, Column column,
+ int columnNumber, int partitionOrder) {
+ rows.add(row("tablename", tableName,
+ "columnname", column.getName(),
+ "external_type", column.getType(),
+ "columnnum", columnNumber,
+ "part_key", partitionOrder,
+ "is_nullable", "true"));
+ }
+
+ private void appendPartition(List<Map<String, Object>> rows, String tableName, Partition partition,
+ StorageDescriptor tableDescriptor) {
+ StorageDescriptor descriptor = partition.getStorageDescriptor();
+ String values = jsonValues(partition.getValues());
+ String serializationLibrary = serdeLibrary(descriptor);
+ if (serializationLibrary.isBlank()) {
+ serializationLibrary = serdeLibrary(tableDescriptor);
+ }
+ Map<String, String> parameters = serdeParameters(descriptor);
+ if (parameters == null) {
+ parameters = serdeParameters(tableDescriptor);
+ }
+ Boolean compressedValue = descriptor == null ? null : descriptor.getCompressed();
+ if (compressedValue == null && tableDescriptor != null) {
+ compressedValue = tableDescriptor.getCompressed();
+ }
+ rows.add(row("tablename", tableName,
+ "values", values,
+ "location", descriptor == null ? "" : descriptor.getLocation(),
+ "input_format", descriptor == null ? "" : descriptor.getInputFormat(),
+ "output_format", descriptor == null ? "" : descriptor.getOutputFormat(),
+ "serialization_lib", serializationLibrary,
+ "serde_parameters", json(parameters),
+ "compressed", Boolean.TRUE.equals(compressedValue) ? 1 : 0,
+ "parameters", json(partition.getParameters())));
+ }
+
+ private static String serdeLibrary(StorageDescriptor descriptor) {
+ return descriptor == null || descriptor.getSerdeInfo() == null
+ || descriptor.getSerdeInfo().getSerializationLibrary() == null
+ ? "" : descriptor.getSerdeInfo().getSerializationLibrary();
+ }
+
+ private static Map<String, String> serdeParameters(StorageDescriptor descriptor) {
+ return descriptor == null || descriptor.getSerdeInfo() == null
+ ? null : descriptor.getSerdeInfo().getParameters();
+ }
+
+ private String json(Object value) {
+ try {
+ return objectMapper.writeValueAsString(value == null ? new LinkedHashMap<>() : value);
+ } catch (JsonProcessingException exception) {
+ throw new IllegalStateException("Unable to serialize external table metadata", exception);
+ }
+ }
+
+ private String jsonValues(List<String> values) {
+ try {
+ return objectMapper.writeValueAsString(values == null ? List.of() : values);
+ } catch (JsonProcessingException exception) {
+ throw new IllegalStateException("Unable to serialize external partition values", exception);
+ }
+ }
+
+ private static String literal(String value) {
+ return "'" + (value == null ? "" : value.replace("'", "''")) + "'";
+ }
+
+ private static Map<String, Object> row(Object... values) {
+ Map<String, Object> result = new LinkedHashMap<>();
+ for (int index = 0; index < values.length; index += 2) {
+ result.put((String) values[index], values[index + 1]);
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScanner.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScanner.java
new file mode 100644
index 0000000000..47834bd1e8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScanner.java
@@ -0,0 +1,456 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Finds references to bound external schemas while ignoring literals and comments.
+ *
+ * <p>A qualified name only counts as a table reference where a table can appear: after
+ * {@code FROM}, {@code JOIN}, {@code USING}, {@code INTO} or {@code UPDATE}, and in a comma-separated
+ * {@code FROM} list. Anywhere else (a select list, {@code ON}, {@code WHERE}) {@code schema.name}
+ * is a column qualifier, so a table aliased with an external schema's name is not mistaken for it.
+ */
+public final class ExternalReferenceScanner {
+
+ /** Keywords after which the following names are table sources. */
+ private static final Set<String> TABLE_INTRODUCERS = Set.of("from", "join", "using", "into", "update");
+
+ /** Functions whose argument list uses FROM as an expression separator, not to start a table list. */
+ private static final Set<String> EXPRESSION_FROM_FUNCTIONS = Set.of("extract", "substring", "trim", "overlay", "position");
+
+ /** Keywords that end a table-source list: what follows is an expression or a new clause. */
+ private static final Set<String> CLAUSE_ENDERS = Set.of("select", "where", "group", "order", "having",
+ "limit", "offset", "union", "intersect", "except", "on", "set", "values", "returning", "window",
+ "fetch", "qualify");
+
+ public record Reference(String schema, String table) {
+ }
+
+ private record Word(String value, boolean quoted) {
+ }
+
+ /** An identifier chain ({@code a.b.c}) or one structural character. */
+ private record Token(List<Word> chain, char symbol) {
+
+ boolean isChain() {
+ return chain != null;
+ }
+
+ boolean isKeyword(String keyword) {
+ return chain != null && chain.size() == 1 && !chain.getFirst().quoted()
+ && chain.getFirst().value().equals(keyword);
+ }
+
+ String keyword() {
+ return chain != null && chain.size() == 1 && !chain.getFirst().quoted() ? chain.getFirst().value() : null;
+ }
+ }
+
+ private ExternalReferenceScanner() {
+ }
+
+ public static Set<Reference> scan(String sql, Set<String> schemaNames) {
+ Set<Reference> result = new LinkedHashSet<>();
+ Deque<Boolean> outerContexts = new ArrayDeque<>();
+ Deque<Boolean> expressionFromParens = new ArrayDeque<>();
+ boolean tableContext = false;
+ String previousKeyword = null;
+ for (Token token : tokenize(sql)) {
+ if (!token.isChain()) {
+ switch (token.symbol()) {
+ case '(' -> {
+ outerContexts.push(tableContext);
+ expressionFromParens.push(previousKeyword != null && EXPRESSION_FROM_FUNCTIONS.contains(previousKeyword));
+ }
+ case ')' -> {
+ tableContext = !outerContexts.isEmpty() && outerContexts.pop();
+ if (!expressionFromParens.isEmpty()) {
+ expressionFromParens.pop();
+ }
+ }
+ case ';' -> {
+ tableContext = false;
+ outerContexts.clear();
+ expressionFromParens.clear();
+ }
+ default -> {
+ // a comma keeps the current context: it separates items of the same list
+ }
+ }
+ previousKeyword = null;
+ continue;
+ }
+ String keyword = token.keyword();
+ boolean expressionFrom = "from".equals(keyword)
+ && ("distinct".equals(previousKeyword) || !expressionFromParens.isEmpty() && expressionFromParens.peek());
+ previousKeyword = keyword;
+ if (expressionFrom) {
+ continue;
+ }
+ if (keyword != null && TABLE_INTRODUCERS.contains(keyword)) {
+ tableContext = true;
+ } else if (keyword != null && CLAUSE_ENDERS.contains(keyword)) {
+ tableContext = false;
+ } else if (tableContext) {
+ Reference reference = reference(token.chain(), schemaNames);
+ if (reference != null) {
+ result.add(reference);
+ }
+ }
+ }
+ return result;
+ }
+
+ public static boolean containsIdentifierPrefix(String sql, String prefix) {
+ if (sql == null || prefix == null) {
+ return false;
+ }
+ return tokenize(sql).stream().filter(Token::isChain).flatMap(token -> token.chain().stream())
+ .anyMatch(word -> word.value().regionMatches(true, 0, prefix, 0, prefix.length()));
+ }
+
+ /**
+ * The first external table or schema an {@code INSERT}, {@code UPDATE}, {@code DELETE}, {@code MERGE},
+ * {@code TRUNCATE}, {@code COPY}, {@code SELECT INTO}, {@code CREATE TABLE}, {@code CREATE VIEW}, {@code CREATE INDEX},
+ * {@code ALTER TABLE} or {@code DROP} anywhere in
+ * {@code sql} changes: every statement of a batch and every data-modifying CTE is checked, not only
+ * the first write. The external DDL Floci implements is parsed before this is reached.
+ */
+ public static Optional<Reference> writeTarget(String sql, Set<String> schemaNames) {
+ if (sql == null) {
+ return Optional.empty();
+ }
+ List<Token> tokens = tokenize(sql);
+ for (int i = 0; i < tokens.size(); i++) {
+ Optional<Reference> moved = movedIntoExternalSchema(tokens, i, schemaNames);
+ if (moved.isPresent()) {
+ return moved;
+ }
+ Optional<Reference> truncated = truncatedTarget(tokens, i, schemaNames);
+ if (truncated.isPresent()) {
+ return truncated;
+ }
+ int target = writeTargetIndex(tokens, i);
+ if (target >= 0 && target < tokens.size() && tokens.get(target).isChain()) {
+ Reference reference = reference(tokens.get(target).chain(), schemaNames);
+ if (reference != null) {
+ return Optional.of(reference);
+ }
+ }
+ Optional<Reference> dropped = droppedTarget(tokens, i, schemaNames);
+ if (dropped.isPresent()) {
+ return dropped;
+ }
+ }
+ return Optional.empty();
+ }
+
+ private static Optional<Reference> movedIntoExternalSchema(List<Token> tokens, int i, Set<String> schemaNames) {
+ if (!keywordAt(tokens, i, "alter") || !keywordAt(tokens, i + 1, "table")) {
+ return Optional.empty();
+ }
+ int source = skipOnly(tokens, i + 2);
+ if (keywordAt(tokens, source, "if") && keywordAt(tokens, source + 1, "exists")) {
+ source += 2;
+ }
+ int operation = source + 1;
+ if (keywordAt(tokens, operation, "rename") && keywordAt(tokens, operation + 1, "to")) {
+ int target = operation + 2;
+ if (target < tokens.size() && tokens.get(target).isChain()) {
+ return Optional.ofNullable(reference(tokens.get(target).chain(), schemaNames));
+ }
+ }
+ if (keywordAt(tokens, operation, "set") && keywordAt(tokens, operation + 1, "schema")) {
+ int target = operation + 2;
+ if (target < tokens.size() && tokens.get(target).isChain()) {
+ Word schema = tokens.get(target).chain().getFirst();
+ if (schemaNames.contains(schema.value())) {
+ return Optional.of(new Reference(schema.value(), ""));
+ }
+ }
+ }
+ return Optional.empty();
+ }
+
+ /** {@code DROP TABLE a, b} and {@code DROP SCHEMA a, b}: any name of the list may be an external one. */
+ private static Optional<Reference> droppedTarget(List<Token> tokens, int i, Set<String> schemaNames) {
+ if (!keywordAt(tokens, i, "drop")) {
+ return Optional.empty();
+ }
+ boolean table = keywordAt(tokens, i + 1, "table");
+ if (!table && !keywordAt(tokens, i + 1, "schema")) {
+ return Optional.empty();
+ }
+ int index = i + 2;
+ if (keywordAt(tokens, index, "if") && keywordAt(tokens, index + 1, "exists")) {
+ index += 2;
+ }
+ while (index < tokens.size() && tokens.get(index).isChain()) {
+ List<Word> chain = tokens.get(index).chain();
+ Reference reference = table ? reference(chain, schemaNames)
+ : schemaNames.contains(chain.getFirst().value()) ? new Reference(chain.getFirst().value(), "") : null;
+ if (reference != null) {
+ return Optional.of(reference);
+ }
+ boolean anotherName = index + 1 < tokens.size() && !tokens.get(index + 1).isChain()
+ && tokens.get(index + 1).symbol() == ',';
+ index += anotherName ? 2 : tokens.size();
+ }
+ return Optional.empty();
+ }
+
+ private static int writeTargetIndex(List<Token> tokens, int i) {
+ if (keywordAt(tokens, i, "create")) {
+ int index = i + 1;
+ while (keywordAt(tokens, index, "temp") || keywordAt(tokens, index, "temporary")
+ || keywordAt(tokens, index, "unlogged") || keywordAt(tokens, index, "local")) {
+ index++;
+ }
+ if (keywordAt(tokens, index, "or") && keywordAt(tokens, index + 1, "replace")) {
+ index += 2;
+ while (keywordAt(tokens, index, "temp") || keywordAt(tokens, index, "temporary")) {
+ index++;
+ }
+ }
+ if (keywordAt(tokens, index, "materialized")) {
+ index++;
+ }
+ if (keywordAt(tokens, index, "unique")) {
+ index++;
+ }
+ if (keywordAt(tokens, index, "index")) {
+ return indexTable(tokens, index + 1);
+ }
+ if (!(keywordAt(tokens, index, "table") || keywordAt(tokens, index, "view"))) {
+ return -1;
+ }
+ index++;
+ if (keywordAt(tokens, index, "if") && keywordAt(tokens, index + 1, "not") && keywordAt(tokens, index + 2, "exists")) {
+ index += 3;
+ }
+ return index;
+ }
+ if (keywordAt(tokens, i, "alter") && keywordAt(tokens, i + 1, "table")) {
+ int index = skipOnly(tokens, i + 2);
+ if (keywordAt(tokens, index, "if") && keywordAt(tokens, index + 1, "exists")) {
+ index += 2;
+ }
+ return skipOnly(tokens, index);
+ }
+ if (keywordAt(tokens, i, "copy")) {
+ return i + 1;
+ }
+ if (keywordAt(tokens, i, "into")) {
+ int index = i + 1;
+ while (keywordAt(tokens, index, "temp") || keywordAt(tokens, index, "temporary")
+ || keywordAt(tokens, index, "unlogged") || keywordAt(tokens, index, "table")) {
+ index++;
+ }
+ return skipOnly(tokens, index);
+ }
+ if (keywordAt(tokens, i, "update")) {
+ return skipOnly(tokens, i + 1);
+ }
+ if (keywordAt(tokens, i, "delete") && keywordAt(tokens, i + 1, "from")) {
+ return skipOnly(tokens, i + 2);
+ }
+ return -1;
+ }
+
+ /** {@code CREATE INDEX [CONCURRENTLY] [IF NOT EXISTS] name ON [ONLY] table}: {@code from} follows INDEX. */
+ private static int indexTable(List<Token> tokens, int from) {
+ for (int index = from; index < from + 6 && index < tokens.size(); index++) {
+ if (keywordAt(tokens, index, "on")) {
+ return skipOnly(tokens, index + 1);
+ }
+ }
+ return -1;
+ }
+
+ private static Optional<Reference> truncatedTarget(List<Token> tokens, int i, Set<String> schemaNames) {
+ if (!keywordAt(tokens, i, "truncate")) {
+ return Optional.empty();
+ }
+ int target = skipOnly(tokens, keywordAt(tokens, i + 1, "table") ? i + 2 : i + 1);
+ while (target < tokens.size() && tokens.get(target).isChain()) {
+ Reference reference = reference(tokens.get(target).chain(), schemaNames);
+ if (reference != null) {
+ return Optional.of(reference);
+ }
+ int comma = target + 1;
+ if (comma >= tokens.size() || tokens.get(comma).isChain() || tokens.get(comma).symbol() != ',') {
+ return Optional.empty();
+ }
+ target = skipOnly(tokens, comma + 1);
+ }
+ return Optional.empty();
+ }
+
+ private static int skipOnly(List<Token> tokens, int index) {
+ return keywordAt(tokens, index, "only") ? index + 1 : index;
+ }
+
+ private static boolean keywordAt(List<Token> tokens, int index, String keyword) {
+ return index < tokens.size() && tokens.get(index).isKeyword(keyword);
+ }
+
+ private static Reference reference(List<Word> chain, Set<String> schemas) {
+ if (chain.size() >= 2 && schemas.contains(chain.get(0).value())) {
+ return new Reference(chain.get(0).value(), chain.get(1).value());
+ }
+ if (chain.size() >= 3 && schemas.contains(chain.get(1).value())) {
+ return new Reference(chain.get(1).value(), chain.get(2).value());
+ }
+ return null;
+ }
+
+ private static List<Token> tokenize(String sql) {
+ List<Token> result = new ArrayList<>();
+ int i = 0;
+ while (i < sql.length()) {
+ char c = sql.charAt(i);
+ if (c == '\'' || c == '-' && i + 1 < sql.length() && sql.charAt(i + 1) == '-'
+ || c == '/' && i + 1 < sql.length() && sql.charAt(i + 1) == '*') {
+ i = skip(sql, i, c == '\'' && isEscapeStringPrefix(sql, i));
+ } else if (c == '$' && dollarTagEnd(sql, i) > 0) {
+ i = skipDollarQuoted(sql, i);
+ } else if (c == '"' || Character.isLetter(c) || c == '_') {
+ List<Word> chain = new ArrayList<>();
+ i = read(sql, i, chain);
+ while (true) {
+ int separator = skipTrivia(sql, i);
+ if (separator >= sql.length() || sql.charAt(separator) != '.') {
+ break;
+ }
+ int next = skipTrivia(sql, separator + 1);
+ if (next >= sql.length() || !(sql.charAt(next) == '"' || Character.isLetter(sql.charAt(next))
+ || sql.charAt(next) == '_')) {
+ break;
+ }
+ i = read(sql, next, chain);
+ }
+ result.add(new Token(chain, '\0'));
+ } else {
+ if (c == '(' || c == ')' || c == ',' || c == ';') {
+ result.add(new Token(null, c));
+ }
+ i++;
+ }
+ }
+ return result;
+ }
+
+ private static int skipTrivia(String sql, int start) {
+ int i = start;
+ while (i < sql.length()) {
+ if (Character.isWhitespace(sql.charAt(i))) {
+ i++;
+ } else if (sql.charAt(i) == '-' && i + 1 < sql.length() && sql.charAt(i + 1) == '-'
+ || sql.charAt(i) == '/' && i + 1 < sql.length() && sql.charAt(i + 1) == '*') {
+ i = skip(sql, i, false);
+ } else {
+ break;
+ }
+ }
+ return i;
+ }
+
+ private static int read(String sql, int start, List<Word> chain) {
+ if (sql.charAt(start) == '"') {
+ StringBuilder value = new StringBuilder();
+ int i = start + 1;
+ while (i < sql.length()) {
+ if (sql.charAt(i) == '"') {
+ if (i + 1 < sql.length() && sql.charAt(i + 1) == '"') {
+ value.append('"');
+ i += 2;
+ } else {
+ i++;
+ break;
+ }
+ } else {
+ value.append(sql.charAt(i++));
+ }
+ }
+ chain.add(new Word(value.toString(), true));
+ return i;
+ }
+ int i = start;
+ while (i < sql.length() && (Character.isLetterOrDigit(sql.charAt(i)) || sql.charAt(i) == '_'
+ || sql.charAt(i) == '$')) {
+ i++;
+ }
+ chain.add(new Word(sql.substring(start, i).toLowerCase(Locale.ROOT), false));
+ return i;
+ }
+
+ private static boolean isEscapeStringPrefix(String sql, int quote) {
+ if (quote == 0 || Character.toLowerCase(sql.charAt(quote - 1)) != 'e') {
+ return false;
+ }
+ if (quote == 1) {
+ return true;
+ }
+ char before = sql.charAt(quote - 2);
+ return !(Character.isLetterOrDigit(before) || before == '_' || before == '$');
+ }
+
+ /** Returns the index just past an opening dollar-quote tag such as {@code $$} or {@code $body$}, or -1. */
+ private static int dollarTagEnd(String sql, int start) {
+ int i = start + 1;
+ if (i < sql.length() && (Character.isLetter(sql.charAt(i)) || sql.charAt(i) == '_')) {
+ while (i < sql.length() && (Character.isLetterOrDigit(sql.charAt(i)) || sql.charAt(i) == '_')) {
+ i++;
+ }
+ }
+ return i < sql.length() && sql.charAt(i) == '$' ? i + 1 : -1;
+ }
+
+ private static int skipDollarQuoted(String sql, int start) {
+ int bodyStart = dollarTagEnd(sql, start);
+ int close = sql.indexOf(sql.substring(start, bodyStart), bodyStart);
+ return close < 0 ? sql.length() : close + (bodyStart - start);
+ }
+
+ private static int skip(String sql, int start, boolean backslashEscapes) {
+ if (sql.charAt(start) == '\'') {
+ int i = start + 1;
+ while (i < sql.length()) {
+ char c = sql.charAt(i);
+ if (backslashEscapes && c == '\\') {
+ i += 2;
+ } else if (c == '\'' && (i + 1 >= sql.length() || sql.charAt(i + 1) != '\'')) {
+ return i + 1;
+ } else {
+ i += c == '\'' ? 2 : 1;
+ }
+ }
+ return sql.length();
+ }
+ if (sql.charAt(start) == '-') {
+ int i = sql.indexOf('\n', start + 2);
+ return i < 0 ? sql.length() : i;
+ }
+ int depth = 1;
+ int i = start + 2;
+ while (i < sql.length() && depth > 0) {
+ if (sql.startsWith("/*", i)) {
+ depth++;
+ i += 2;
+ } else if (sql.startsWith("*/", i)) {
+ depth--;
+ i += 2;
+ } else {
+ i++;
+ }
+ }
+ return Math.min(i, sql.length());
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalSchemaBinding.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalSchemaBinding.java
new file mode 100644
index 0000000000..d935dab68c
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalSchemaBinding.java
@@ -0,0 +1,22 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+public record ExternalSchemaBinding(String accountId, String clusterKey, String databaseName,
+ String schemaName, String glueDatabase, String iamRoleArn) {
+ public ExternalSchemaBinding {
+ require(accountId, "accountId");
+ require(clusterKey, "clusterKey");
+ require(databaseName, "databaseName");
+ require(schemaName, "schemaName");
+ require(glueDatabase, "glueDatabase");
+ require(iamRoleArn, "iamRoleArn");
+ }
+
+ private static void require(String value, String field) {
+ if (value == null || value.isBlank()) {
+ throw new IllegalArgumentException(field + " is required");
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatement.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatement.java
new file mode 100644
index 0000000000..096ccfe818
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatement.java
@@ -0,0 +1,46 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public sealed interface ExternalStatement permits ExternalStatement.CreateSchema, ExternalStatement.CreateTable,
+ ExternalStatement.AddPartitions, ExternalStatement.DropSchema, ExternalStatement.DropTable {
+ /** Read-only copy that keeps statement order, so Glue payloads are stable across runs. */
+ private static <V> Map<String, V> ordered(Map<String, V> source) {
+ return Collections.unmodifiableMap(new LinkedHashMap<>(source));
+ }
+
+ record ColumnDefinition(String name, String type) { }
+ enum TableFormat { PARQUET, TEXTFILE, JSON }
+ record PartitionSpec(Map<String, String> values, String location) {
+ public PartitionSpec { values = ordered(values); }
+ }
+ record CreateSchema(String schemaName, String glueDatabase, String iamRoleArn, boolean createDatabaseIfNotExists,
+ boolean ifNotExists) implements ExternalStatement { }
+ record CreateTable(String schemaName, String tableName, List<ColumnDefinition> columns,
+ List<ColumnDefinition> partitionColumns, TableFormat format, String location,
+ String delimiter, String serde, Map<String, String> properties,
+ Map<String, String> serdeProperties) implements ExternalStatement {
+ public CreateTable {
+ columns = List.copyOf(columns);
+ partitionColumns = List.copyOf(partitionColumns);
+ properties = ordered(properties);
+ serdeProperties = ordered(serdeProperties);
+ }
+
+ public CreateTable(String schemaName, String tableName, List<ColumnDefinition> columns,
+ List<ColumnDefinition> partitionColumns, TableFormat format, String location,
+ String delimiter, String serde, Map<String, String> properties) {
+ this(schemaName, tableName, columns, partitionColumns, format, location, delimiter, serde, properties, Map.of());
+ }
+ }
+ record AddPartitions(String schemaName, String tableName, boolean ifNotExists,
+ List<PartitionSpec> partitions) implements ExternalStatement {
+ public AddPartitions { partitions = List.copyOf(partitions); }
+ }
+ /** {@code cascade} is true only when the statement said CASCADE; RESTRICT is the default, as in PostgreSQL and Redshift. */
+ record DropSchema(String schemaName, boolean ifExists, boolean cascade) implements ExternalStatement { }
+ record DropTable(String schemaName, String tableName, boolean ifExists, boolean cascade) implements ExternalStatement { }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParser.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParser.java
new file mode 100644
index 0000000000..a4a41bf237
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParser.java
@@ -0,0 +1,323 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import jakarta.enterprise.context.ApplicationScoped;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@ApplicationScoped
+public class ExternalStatementParser {
+ private static final Pattern CREATE_SCHEMA = Pattern.compile("(?is)^\\s*CREATE\\s+EXTERNAL\\s+SCHEMA\\s+(IF\\s+NOT\\s+EXISTS\\s+)?(.+?)\\s+FROM\\s+DATA\\s+CATALOG\\s+DATABASE\\s+'((?:''|[^'])*)'(?:\\s+REGION\\s+'(?:''|[^'])*')?\\s+IAM_ROLE\\s+(.+?)(\\s+CREATE\\s+EXTERNAL\\s+DATABASE\\s+IF\\s+NOT\\s+EXISTS)?\\s*;?\\s*$");
+ private static final Pattern CREATE_EXTERNAL_TABLE = Pattern.compile("(?is)^\\s*CREATE\\s+EXTERNAL\\s+TABLE\\b.*");
+ private static final Pattern CREATE_EXTERNAL = Pattern.compile("(?is)^\\s*CREATE\\s+EXTERNAL\\b.*");
+ private static final Pattern SERDE_PROPERTIES = Pattern.compile("(?is)WITH\\s+SERDEPROPERTIES\\s*\\(");
+ private static final String NAME = "(\"(?:\"\"|[^\"])+\"|[^.;,\\s\"()]+)";
+ private static final Pattern CREATE_TABLE_HEAD = Pattern.compile("(?is)^\\s*CREATE\\s+EXTERNAL\\s+TABLE\\s+" + NAME + "\\s*\\.\\s*" + NAME + "\\s*\\(");
+ private static final Pattern DROP_TABLE = Pattern.compile("(?is)^\\s*DROP\\s+TABLE\\s+(IF\\s+EXISTS\\s+)?" + NAME + "\\.\\s*" + NAME + "(?:\\s+(CASCADE|RESTRICT))?\\s*;?\\s*$");
+ private static final Pattern DROP_SCHEMA = Pattern.compile("(?is)^\\s*DROP\\s+SCHEMA\\s+(IF\\s+EXISTS\\s+)?" + NAME + "(?:\\s+(CASCADE|RESTRICT))?\\s*;?\\s*$");
+ private static final Pattern TABLE_PROPERTIES = Pattern.compile("(?is)(?:TBLPROPERTIES|TABLE\\s+PROPERTIES)\\s*\\(");
+ private static final Pattern PARTITION_VALUE = Pattern.compile("(?is)([^,=]+)=\\s*'((?:''|[^'])*)'\\s*(?:,|$)");
+ private static final Pattern LOCATION_VALUE = Pattern.compile("(?is)LOCATION\\s+'((?:''|[^'])*)'");
+ private static final Pattern SERDE_VALUE = Pattern.compile("(?is)SERDE\\s+'((?:''|[^'])*)'");
+ private static final Pattern TERMINATED_BY_VALUE = Pattern.compile("(?is)TERMINATED BY\\s+'((?:''|[^'])*)'");
+ private static final Pattern STORED_AS_VALUE = Pattern.compile("(?is)STORED AS\\s+([A-Za-z]+)");
+ private static final Pattern QUOTED_LITERAL = Pattern.compile("'(?:''|[^'])*'");
+ private static final Pattern QUOTED_PAIR = Pattern.compile("'((?:''|[^'])*)'\\s*=\\s*'((?:''|[^'])*)'");
+ private static final Pattern PARTITION_HEAD = Pattern.compile("(?is)PARTITION\\s*\\(");
+ private static final Pattern PARTITION_LOCATION = Pattern.compile("(?is)\\s*LOCATION\\s+'((?:''|[^'])*)'");
+ private static final Pattern PARTITIONED_BY = Pattern.compile("(?is)PARTITIONED\\s+BY\\s*\\(");
+ private static final Pattern ADD_PARTITIONS = Pattern.compile("(?is)^\\s*ALTER\\s+TABLE\\s+" + NAME + "\\.\\s*" + NAME + "\\s+ADD\\s+(IF\\s+NOT\\s+EXISTS\\s+)?(.+?)\\s*;?\\s*$");
+
+ public Optional<ExternalStatement> parse(String sql) {
+ if (sql == null) {
+ return Optional.empty();
+ }
+ String text = stripLeadingComments(sql);
+ Matcher schema = CREATE_SCHEMA.matcher(text);
+ if (schema.matches()) {
+ String role = schema.group(4).trim();
+ if (role.equalsIgnoreCase("default")) {
+ throw new SpectrumSqlException("0A000", "IAM_ROLE DEFAULT is not supported");
+ }
+ if (!QUOTED_LITERAL.matcher(role).matches()) {
+ throw new SpectrumSqlException("42601", "IAM_ROLE must be a single quoted role ARN");
+ }
+ return Optional.of(new ExternalStatement.CreateSchema(identifier(schema.group(2)),
+ schema.group(3).replace("''", "'"), unquote(role), schema.group(5) != null,
+ schema.group(1) != null));
+ }
+ if (CREATE_EXTERNAL_TABLE.matcher(text).matches()) {
+ return Optional.of(parseTable(text));
+ }
+ if (CREATE_EXTERNAL.matcher(text).matches()) {
+ throw new SpectrumSqlException("0A000", "unsupported CREATE EXTERNAL statement");
+ }
+ Matcher dropTable = DROP_TABLE.matcher(text);
+ if (dropTable.matches()) {
+ return Optional.of(new ExternalStatement.DropTable(identifier(dropTable.group(2)), identifier(dropTable.group(3)),
+ dropTable.group(1) != null, "CASCADE".equalsIgnoreCase(dropTable.group(4))));
+ }
+ Matcher dropSchema = DROP_SCHEMA.matcher(text);
+ if (dropSchema.matches()) {
+ return Optional.of(new ExternalStatement.DropSchema(identifier(dropSchema.group(2)),
+ dropSchema.group(1) != null, "CASCADE".equalsIgnoreCase(dropSchema.group(3))));
+ }
+ Matcher addPartitions = ADD_PARTITIONS.matcher(text);
+ if (addPartitions.matches()) {
+ List<ExternalStatement.PartitionSpec> partitions = parsePartitions(addPartitions.group(4));
+ return partitions.isEmpty() ? Optional.empty() : Optional.of(new ExternalStatement.AddPartitions(
+ identifier(addPartitions.group(1)), identifier(addPartitions.group(2)),
+ addPartitions.group(3) != null, partitions));
+ }
+ return Optional.empty();
+ }
+
+ private ExternalStatement.CreateTable parseTable(String text) {
+ Matcher matcher = CREATE_TABLE_HEAD.matcher(text);
+ if (!matcher.find()) {
+ throw new SpectrumSqlException("0A000", "malformed CREATE EXTERNAL TABLE");
+ }
+ int open = matcher.end() - 1;
+ int close = matchingParen(text, open);
+ if (close < 0) {
+ throw new SpectrumSqlException("0A000", "malformed CREATE EXTERNAL TABLE column list");
+ }
+ String tail = text.substring(close + 1);
+ List<ExternalStatement.ColumnDefinition> columns = columns(text.substring(open + 1, close));
+ List<ExternalStatement.ColumnDefinition> partitions = List.of();
+ Matcher partition = PARTITIONED_BY.matcher(tail);
+ if (partition.find()) {
+ // Partition columns may have types such as decimal(10,2), so match nested parentheses.
+ int partitionOpen = partition.end() - 1;
+ int partitionClose = matchingParen(tail, partitionOpen);
+ if (partitionClose < 0) {
+ throw new SpectrumSqlException("0A000", "malformed CREATE EXTERNAL TABLE partition list");
+ }
+ partitions = columns(tail.substring(partitionOpen + 1, partitionClose));
+ }
+ String location = value(tail, LOCATION_VALUE);
+ if (location == null || !location.startsWith("s3://")) {
+ throw new SpectrumSqlException("22023", "external table location must be an s3:// URI");
+ }
+ String serde = value(tail, SERDE_VALUE);
+ String delimiter = value(tail, TERMINATED_BY_VALUE);
+ String stored = keywordValue(tail, STORED_AS_VALUE);
+ if (stored == null) {
+ throw new SpectrumSqlException("0A000", "external table requires a STORED AS clause");
+ }
+ if (!(stored.equalsIgnoreCase("PARQUET") || stored.equalsIgnoreCase("TEXTFILE"))) {
+ throw new SpectrumSqlException("0A000", "unsupported external table format: " + stored);
+ }
+ ExternalStatement.TableFormat format = stored.equalsIgnoreCase("PARQUET")
+ ? ExternalStatement.TableFormat.PARQUET
+ : serde != null && serde.toLowerCase(Locale.ROOT).contains("json")
+ ? ExternalStatement.TableFormat.JSON : ExternalStatement.TableFormat.TEXTFILE;
+ Map<String, String> properties = properties(tail);
+ return new ExternalStatement.CreateTable(identifier(matcher.group(1)), identifier(matcher.group(2)), columns, partitions,
+ format, location, delimiter, serde, properties, serdeProperties(tail));
+ }
+
+ private static String stripLeadingComments(String sql) {
+ String text = sql.trim();
+ while (true) {
+ int end;
+ if (text.startsWith("--")) {
+ end = text.indexOf('\n');
+ end = end < 0 ? text.length() : end;
+ } else if (text.startsWith("/*")) {
+ end = text.indexOf("*/", 2);
+ end = end < 0 ? text.length() : end + 2;
+ } else {
+ return text;
+ }
+ text = text.substring(end).trim();
+ }
+ }
+
+ private static int matchingParen(String text, int open) {
+ int depth = 0;
+ char quote = 0;
+ for (int i = open; i < text.length(); i++) {
+ char c = text.charAt(i);
+ if (quote != 0) {
+ if (c == quote) {
+ if (i + 1 < text.length() && text.charAt(i + 1) == quote) {
+ i++;
+ } else {
+ quote = 0;
+ }
+ }
+ } else if (c == '\'' || c == '"') {
+ quote = c;
+ } else if (c == '(') {
+ depth++;
+ } else if (c == ')' && --depth == 0) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ private static List<ExternalStatement.ColumnDefinition> columns(String text) {
+ List<ExternalStatement.ColumnDefinition> result = new ArrayList<>();
+ for (String item : split(text)) {
+ String definition = item.trim();
+ int nameEnd = columnNameEnd(definition);
+ if (nameEnd < 0 || nameEnd == definition.length() || definition.substring(nameEnd).isBlank()) {
+ throw new SpectrumSqlException("0A000", "malformed column definition");
+ }
+ result.add(new ExternalStatement.ColumnDefinition(identifier(definition.substring(0, nameEnd)),
+ GlueTypeMapper.canonicalGlueType(definition.substring(nameEnd).trim())));
+ }
+ return result;
+ }
+
+ private static int columnNameEnd(String definition) {
+ if (!definition.startsWith("\"")) {
+ for (int index = 0; index < definition.length(); index++) {
+ if (Character.isWhitespace(definition.charAt(index))) {
+ return index;
+ }
+ }
+ return -1;
+ }
+ for (int index = 1; index < definition.length(); index++) {
+ if (definition.charAt(index) == '"') {
+ if (index + 1 < definition.length() && definition.charAt(index + 1) == '"') {
+ index++;
+ } else {
+ return index + 1;
+ }
+ }
+ }
+ return -1;
+ }
+
+ private static List<String> split(String text) {
+ List<String> result = new ArrayList<>();
+ int depth = 0;
+ int start = 0;
+ char quote = 0;
+ for (int i = 0; i < text.length(); i++) {
+ char c = text.charAt(i);
+ if (quote != 0) {
+ if (c == quote) {
+ if (i + 1 < text.length() && text.charAt(i + 1) == quote) {
+ i++;
+ } else {
+ quote = 0;
+ }
+ }
+ } else if (c == '\'' || c == '"') {
+ quote = c;
+ } else if (c == '(' || c == '<') {
+ depth++;
+ } else if (c == ')' || c == '>') {
+ depth--;
+ } else if (c == ',' && depth == 0) {
+ result.add(text.substring(start, i));
+ start = i + 1;
+ }
+ }
+ result.add(text.substring(start));
+ return result;
+ }
+
+ private static String value(String text, Pattern pattern) {
+ Matcher matcher = pattern.matcher(text);
+ return matcher.find() ? matcher.group(1).replace("''", "'") : null;
+ }
+
+ private static String keywordValue(String text, Pattern pattern) {
+ Matcher matcher = pattern.matcher(text);
+ return matcher.find() ? matcher.group(1) : null;
+ }
+
+ private static Map<String, String> serdeProperties(String text) {
+ Map<String, String> result = new LinkedHashMap<>();
+ Matcher matcher = SERDE_PROPERTIES.matcher(text);
+ if (matcher.find()) {
+ int open = matcher.end() - 1;
+ int close = matchingParen(text, open);
+ if (close < 0) {
+ throw new SpectrumSqlException("0A000", "malformed SERDEPROPERTIES clause");
+ }
+ result.putAll(quotedPairs(text.substring(open + 1, close)));
+ }
+ return result;
+ }
+
+ private static Map<String, String> properties(String text) {
+ Map<String, String> result = new LinkedHashMap<>();
+ Matcher matcher = TABLE_PROPERTIES.matcher(text);
+ if (matcher.find()) {
+ int open = matcher.end() - 1;
+ int close = matchingParen(text, open);
+ if (close < 0) {
+ throw new SpectrumSqlException("0A000", "malformed TABLE PROPERTIES clause");
+ }
+ result.putAll(quotedPairs(text.substring(open + 1, close)));
+ }
+ return result;
+ }
+
+ private static Map<String, String> quotedPairs(String text) {
+ Map<String, String> result = new LinkedHashMap<>();
+ Matcher pair = QUOTED_PAIR.matcher(text);
+ while (pair.find()) {
+ result.put(pair.group(1).replace("''", "'"), pair.group(2).replace("''", "'"));
+ }
+ return result;
+ }
+
+ private static List<ExternalStatement.PartitionSpec> parsePartitions(String text) {
+ List<ExternalStatement.PartitionSpec> result = new ArrayList<>();
+ Matcher head = PARTITION_HEAD.matcher(text);
+ int end = 0;
+ while (head.find(end)) {
+ if (!text.substring(end, head.start()).isBlank()) {
+ return List.of();
+ }
+ int open = head.end() - 1;
+ int close = matchingParen(text, open);
+ if (close < 0) {
+ return List.of();
+ }
+ Matcher location = PARTITION_LOCATION.matcher(text).region(close + 1, text.length());
+ if (!location.lookingAt()) {
+ return List.of();
+ }
+ Map<String, String> values = new LinkedHashMap<>();
+ Matcher value = PARTITION_VALUE.matcher(text.substring(open + 1, close).trim() + ",");
+ while (value.find()) {
+ values.put(identifier(value.group(1)), value.group(2).replace("''", "'"));
+ }
+ if (values.isEmpty()) {
+ return List.of();
+ }
+ result.add(new ExternalStatement.PartitionSpec(values, location.group(1).replace("''", "'")));
+ end = location.end();
+ }
+ return end == text.length() || text.substring(end).isBlank() ? result : List.of();
+ }
+
+ private static String identifier(String value) {
+ String trimmed = value.trim();
+ if (trimmed.startsWith("\"") && trimmed.endsWith("\"")) {
+ return trimmed.substring(1, trimmed.length() - 1).replace("\"\"", "\"");
+ }
+ return trimmed.toLowerCase(Locale.ROOT);
+ }
+
+ private static String unquote(String value) {
+ String trimmed = value.trim();
+ return trimmed.startsWith("'") && trimmed.endsWith("'")
+ ? trimmed.substring(1, trimmed.length() - 1).replace("''", "'") : trimmed;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilder.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilder.java
new file mode 100644
index 0000000000..d162c4bee5
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilder.java
@@ -0,0 +1,115 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+public final class GlueTableBuilder {
+ private static final String TEXT_INPUT_FORMAT = "org.apache.hadoop.mapred.TextInputFormat";
+ private static final String TEXT_OUTPUT_FORMAT = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat";
+ private static final String LAZY_SIMPLE_SERDE = "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
+ private static final String OPEN_CSV_SERDE = "org.apache.hadoop.hive.serde2.OpenCSVSerde";
+
+ private GlueTableBuilder() {
+ }
+
+ public static Table toGlueTable(ExternalStatement.CreateTable statement) {
+ StorageDescriptor descriptor = new StorageDescriptor();
+ descriptor.setLocation(statement.location());
+ descriptor.setColumns(columns(statement.columns()));
+ StorageDescriptor.SerDeInfo serde = new StorageDescriptor.SerDeInfo();
+ Map<String, String> serdeParameters = new LinkedHashMap<>();
+ switch (statement.format()) {
+ case PARQUET -> {
+ descriptor.setInputFormat("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat");
+ descriptor.setOutputFormat("org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat");
+ serde.setSerializationLibrary("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe");
+ }
+ case JSON -> {
+ descriptor.setInputFormat(TEXT_INPUT_FORMAT);
+ descriptor.setOutputFormat(TEXT_OUTPUT_FORMAT);
+ serde.setSerializationLibrary(statement.serde() == null ? "org.openx.data.jsonserde.JsonSerDe" : statement.serde());
+ }
+ case TEXTFILE -> {
+ descriptor.setInputFormat(TEXT_INPUT_FORMAT);
+ descriptor.setOutputFormat(TEXT_OUTPUT_FORMAT);
+ if (isOpenCsv(statement.serde())) {
+ // OpenCSVSerde takes its delimiter from separatorChar, which the SERDEPROPERTIES supply
+ serde.setSerializationLibrary(OPEN_CSV_SERDE);
+ } else {
+ serde.setSerializationLibrary(LAZY_SIMPLE_SERDE);
+ String delimiter = statement.delimiter() == null ? "," : statement.delimiter();
+ serdeParameters.put("field.delim", delimiter);
+ serdeParameters.put("serialization.format", delimiter);
+ }
+ }
+ }
+ serdeParameters.putAll(statement.serdeProperties());
+ serde.setParameters(serdeParameters);
+ descriptor.setSerdeInfo(serde);
+ Table table = new Table();
+ table.setName(statement.tableName());
+ table.setTableType("EXTERNAL_TABLE");
+ table.setStorageDescriptor(descriptor);
+ table.setPartitionKeys(columns(statement.partitionColumns()));
+ Map<String, String> parameters = new LinkedHashMap<>(statement.properties());
+ parameters.put("EXTERNAL", "TRUE");
+ table.setParameters(parameters);
+ return table;
+ }
+
+ public static Partition toGluePartition(Table table, ExternalStatement.PartitionSpec spec) {
+ if (spec.location() == null || !spec.location().startsWith("s3://")) {
+ throw new SpectrumSqlException("22023", "partition location must be an s3:// URI");
+ }
+ List<String> values = new ArrayList<>();
+ for (Column key : table.getPartitionKeys()) {
+ String value = spec.values().entrySet().stream()
+ .filter(entry -> entry.getKey().equalsIgnoreCase(key.getName()))
+ .map(Map.Entry::getValue).findFirst().orElse(null);
+ if (value == null) {
+ throw new SpectrumSqlException("22023", "partition key \"" + key.getName() + "\" is missing");
+ }
+ values.add(value);
+ }
+ for (String name : spec.values().keySet()) {
+ boolean declared = table.getPartitionKeys().stream().anyMatch(key -> key.getName().equalsIgnoreCase(name));
+ if (!declared) {
+ throw new SpectrumSqlException("22023", "\"" + name + "\" is not a partition key of the table");
+ }
+ }
+ StorageDescriptor source = table.getStorageDescriptor();
+ StorageDescriptor descriptor = new StorageDescriptor();
+ descriptor.setColumns(source.getColumns());
+ descriptor.setInputFormat(source.getInputFormat());
+ descriptor.setOutputFormat(source.getOutputFormat());
+ descriptor.setSerdeInfo(source.getSerdeInfo());
+ descriptor.setLocation(spec.location());
+ Partition partition = new Partition();
+ partition.setValues(values);
+ partition.setStorageDescriptor(descriptor);
+ return partition;
+ }
+
+ private static boolean isOpenCsv(String serde) {
+ return serde != null && serde.toLowerCase(Locale.ROOT).contains("opencsvserde");
+ }
+
+ private static List<Column> columns(List<ExternalStatement.ColumnDefinition> definitions) {
+ List<Column> result = new ArrayList<>();
+ for (ExternalStatement.ColumnDefinition definition : definitions) {
+ Column column = new Column();
+ column.setName(definition.name());
+ column.setType(definition.type().toLowerCase(Locale.ROOT));
+ result.add(column);
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapper.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapper.java
new file mode 100644
index 0000000000..92a2305162
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapper.java
@@ -0,0 +1,111 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** Maps Glue column types to PostgreSQL types and DuckDB text projections. */
+public final class GlueTypeMapper {
+ private static final Pattern SIZED_STRING = Pattern.compile("(?:varchar|char)\\((\\d+)\\)");
+ private static final Pattern DECIMAL = Pattern.compile("(?:decimal|numeric)(?:\\((\\d+)(?:,(\\d+))?\\))?");
+ private static final String NULL_MARKER = "'\\N'";
+
+ private GlueTypeMapper() {
+ }
+
+ /** Rewrites Redshift type aliases to the Hive type names that Glue stores; other types pass through lowercased. */
+ public static String canonicalGlueType(String type) {
+ String normalized = normalize(type);
+ if (isTypeOrSized(normalized, "charactervarying")) {
+ String length = normalized.substring("charactervarying".length());
+ return "varchar" + (length.isEmpty() ? "(256)" : length);
+ }
+ if (isTypeOrSized(normalized, "character")) {
+ String length = normalized.substring("character".length());
+ return "char" + (length.isEmpty() ? "(1)" : length);
+ }
+ if (isTypeOrSized(normalized, "numeric")) {
+ return "decimal" + normalized.substring("numeric".length());
+ }
+ if (isBinary(normalized)) {
+ return "binary";
+ }
+ return switch (normalized) {
+ case "varchar" -> "varchar(256)";
+ case "char" -> "char(1)";
+ case "doubleprecision", "float8" -> "double";
+ case "real", "float4" -> "float";
+ case "int2" -> "smallint";
+ case "int4" -> "int";
+ case "int8" -> "bigint";
+ case "bool" -> "boolean";
+ default -> normalized;
+ };
+ }
+
+ public static String toPostgres(String glueType) {
+ String type = canonicalGlueType(glueType);
+ Matcher sized = SIZED_STRING.matcher(type);
+ if (sized.matches()) {
+ return "varchar(" + sized.group(1) + ")";
+ }
+ Matcher decimal = DECIMAL.matcher(type);
+ if (decimal.matches()) {
+ if (decimal.group(1) == null) {
+ return "numeric";
+ }
+ return "numeric(" + decimal.group(1) + "," + (decimal.group(2) == null ? "0" : decimal.group(2)) + ")";
+ }
+ if (isNested(type)) {
+ return "jsonb";
+ }
+ return switch (type) {
+ case "string", "varchar", "char" -> "text";
+ case "tinyint", "smallint" -> "smallint";
+ case "int", "integer" -> "integer";
+ case "bigint" -> "bigint";
+ case "float" -> "real";
+ case "double" -> "double precision";
+ case "boolean" -> "boolean";
+ case "date" -> "date";
+ case "timestamp" -> "timestamp";
+ case "binary" -> "bytea";
+ default -> throw new SpectrumSqlException("0A000", "Unsupported external column type: " + glueType);
+ };
+ }
+
+ public static boolean isNested(String glueType) {
+ String type = normalize(glueType);
+ return type.startsWith("array<") || type.startsWith("map<") || type.startsWith("struct<");
+ }
+
+ public static String duckProjection(String columnName, String glueType) {
+ String column = quote(columnName);
+ String type = canonicalGlueType(glueType);
+ String expression = isNested(type) ? "to_json(" + column + ")"
+ : "binary".equals(type) ? "'\\x' || hex(" + column + ")" : column;
+ return "COALESCE(CAST(" + expression + " AS VARCHAR), " + NULL_MARKER + ") AS " + column;
+ }
+
+ private static boolean isTypeOrSized(String type, String base) {
+ return type.equals(base) || type.startsWith(base + "(");
+ }
+
+ private static boolean isBinary(String type) {
+ return isTypeOrSized(type, "varbyte")
+ || isTypeOrSized(type, "varbinary")
+ || isTypeOrSized(type, "binaryvarying")
+ || isTypeOrSized(type, "binary");
+ }
+
+ private static String normalize(String glueType) {
+ if (glueType == null) {
+ throw new SpectrumSqlException("0A000", "External column type is missing");
+ }
+ return glueType.trim().toLowerCase(Locale.ROOT).replace(" ", "");
+ }
+
+ private static String quote(String identifier) {
+ return "\"" + identifier.replace("\"", "\"\"") + "\"";
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSession.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSession.java
new file mode 100644
index 0000000000..b0073ff3df
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSession.java
@@ -0,0 +1,173 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import io.github.hectorvent.floci.services.redshift.proxy.PostgresWireDecoder;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+
+public final class PostgresBackendSession implements BackendSql {
+ private static final String SQLSTATE_IO = "58030";
+ private final Socket backend;
+
+ public PostgresBackendSession(Socket backend) {
+ this.backend = backend;
+ }
+
+ @Override
+ public void execute(String sql) {
+ try {
+ OutputStream output = backend.getOutputStream();
+ output.write(PostgresWireDecoder.encodeQuery(sql));
+ output.flush();
+ awaitReady(new PostgresWireDecoder(backend.getInputStream()));
+ } catch (IOException exception) {
+ throw new SpectrumReadException(SQLSTATE_IO, "Unable to run a statement on the Redshift backend", exception);
+ }
+ }
+
+ @Override
+ public long copyIn(String copySql, InputStream data) {
+ try {
+ OutputStream output = backend.getOutputStream();
+ output.write(PostgresWireDecoder.encodeQuery(copySql));
+ output.flush();
+ PostgresWireDecoder decoder = new PostgresWireDecoder(backend.getInputStream());
+ awaitCopyIn(decoder);
+ byte[] chunk = new byte[8192];
+ int read = readChunk(output, decoder, data, chunk);
+ while (read > 0) {
+ output.write('d');
+ writeInt32(output, read + 4);
+ output.write(chunk, 0, read);
+ read = readChunk(output, decoder, data, chunk);
+ }
+ output.write('c');
+ writeInt32(output, 4);
+ output.flush();
+ return rowCount(awaitReady(decoder));
+ } catch (IOException exception) {
+ throw new SpectrumReadException(SQLSTATE_IO, "Unable to load external table data", exception);
+ }
+ }
+
+ /** A failure reading the source aborts the COPY on the backend; a failure writing to the backend socket does not. */
+ private static int readChunk(OutputStream output, PostgresWireDecoder decoder, InputStream data, byte[] chunk)
+ throws IOException {
+ try {
+ int read = data.read(chunk);
+ if (read != 0) {
+ return read;
+ }
+ int nextByte = data.read();
+ if (nextByte < 0) {
+ return -1;
+ }
+ chunk[0] = (byte) nextByte;
+ return 1;
+ } catch (IOException exception) {
+ writeCopyFail(output, exception.getMessage());
+ try {
+ awaitReady(decoder);
+ } catch (IOException | SpectrumReadException backendFailure) {
+ exception.addSuppressed(backendFailure);
+ }
+ throw exception;
+ }
+ }
+
+ private static void awaitCopyIn(PostgresWireDecoder decoder) throws IOException {
+ SpectrumReadException failure = null;
+ PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();
+ while (message != null) {
+ if (message.type() == 'G') {
+ if (failure != null) {
+ throw failure;
+ }
+ return;
+ }
+ if (message.type() == 'E') {
+ failure = toException(message.body());
+ } else if (message.type() == 'Z') {
+ if (failure != null) {
+ throw failure;
+ }
+ throw new IOException("PostgreSQL did not enter CopyIn mode");
+ }
+ message = decoder.nextMessage();
+ }
+ throw new IOException("PostgreSQL closed before CopyInResponse");
+ }
+
+ private static String awaitReady(PostgresWireDecoder decoder) throws IOException {
+ SpectrumReadException failure = null;
+ String tag = "";
+ PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();
+ while (message != null) {
+ if (message.type() == 'C') {
+ tag = new String(message.body(), StandardCharsets.UTF_8).trim();
+ } else if (message.type() == 'E') {
+ failure = toException(message.body());
+ } else if (message.type() == 'Z') {
+ if (failure != null) {
+ throw failure;
+ }
+ return tag;
+ }
+ message = decoder.nextMessage();
+ }
+ throw new IOException("PostgreSQL closed before ReadyForQuery");
+ }
+
+ private static long rowCount(String tag) {
+ int space = tag.lastIndexOf(' ');
+ if (space < 0) {
+ return 0;
+ }
+ try {
+ return Long.parseLong(tag.substring(space + 1).trim());
+ } catch (NumberFormatException exception) {
+ throw new SpectrumReadException(SQLSTATE_IO, "Unexpected COPY command tag: " + tag, exception);
+ }
+ }
+
+ private static SpectrumReadException toException(byte[] body) {
+ String state = SQLSTATE_IO;
+ String message = "PostgreSQL rejected the statement";
+ int offset = 0;
+ while (body != null && offset < body.length && body[offset] != 0) {
+ char code = (char) body[offset++];
+ int end = offset;
+ while (end < body.length && body[end] != 0) {
+ end++;
+ }
+ String value = new String(body, offset, end - offset, StandardCharsets.UTF_8);
+ if (code == 'C') {
+ state = value;
+ } else if (code == 'M') {
+ message = value;
+ }
+ offset = end + 1;
+ }
+ return new SpectrumReadException(state, message);
+ }
+
+ private static void writeInt32(OutputStream output, int value) throws IOException {
+ output.write((value >>> 24) & 0xff);
+ output.write((value >>> 16) & 0xff);
+ output.write((value >>> 8) & 0xff);
+ output.write(value & 0xff);
+ }
+
+ private static void writeCopyFail(OutputStream output, String message) throws IOException {
+ byte[] detail = (message == null ? "Unable to stream COPY input" : message)
+ .getBytes(StandardCharsets.UTF_8);
+ output.write('f');
+ writeInt32(output, detail.length + 5);
+ output.write(detail);
+ output.write(0);
+ output.flush();
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumColumn.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumColumn.java
index 5427eae299..3274e72f84 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumColumn.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumColumn.java
@@ -50,7 +50,7 @@ public record SpectrumColumn(String name, Type type) {
public String postgresType() {
return switch (this) {
- case VARCHAR -> "VARCHAR";
+ case VARCHAR -> "VARCHAR(256)";
case CHAR -> "CHAR";
case INTEGER -> "INTEGER";
case BIGINT -> "BIGINT";
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java
index 5d38129689..a6a0384882 100644
--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java
@@ -21,6 +21,7 @@ public final class SpectrumMaterializer {
private static final SecureRandom RANDOM = new SecureRandom();
private static final String SQLSTATE_DATA = "22000";
+ private static final int DEFAULT_VARCHAR_MAX_BYTES = 256;
public Materialization materialize(Socket backend, SpectrumExternalTable table,
SpectrumExternalSchema schema, SpectrumS3Reader reader) {
@@ -44,7 +45,7 @@ public final class SpectrumMaterializer {
try (Stream<SpectrumRow> rows = reader.read(schema, table)) {
Iterator<SpectrumRow> iterator = rows.iterator();
while (iterator.hasNext()) {
- writeCopyData(output, encodeRow(iterator.next()));
+ writeCopyData(output, encodeRow(iterator.next(), table.columns()));
}
}
writeCopyDone(output);
@@ -96,13 +97,18 @@ public final class SpectrumMaterializer {
+ ") FROM STDIN";
}
- private static byte[] encodeRow(SpectrumRow row) {
+ private static byte[] encodeRow(SpectrumRow row, List<SpectrumColumn> columns) {
StringBuilder line = new StringBuilder();
for (int i = 0; i < row.values().size(); i++) {
if (i > 0) {
line.append('\t');
}
String value = row.values().get(i);
+ if (value != null
+ && columns.get(i).type() == SpectrumColumn.Type.VARCHAR
+ && value.getBytes(StandardCharsets.UTF_8).length > DEFAULT_VARCHAR_MAX_BYTES) {
+ value = null;
+ }
if (value == null) {
line.append("\\N");
} else {
diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumSession.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumSession.java
new file mode 100644
index 0000000000..c9d9c83577
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumSession.java
@@ -0,0 +1,10 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import java.util.List;
+
+public record SpectrumSession(String accountId, String clusterKey, String databaseName,
+ List<String> iamRoleArns, boolean inTransaction) {
+ public SpectrumSession {
+ iamRoleArns = iamRoleArns == null ? List.of() : List.copyOf(iamRoleArns);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistryTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistryTest.java
new file mode 100644
index 0000000000..2436fb113a
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistryTest.java
@@ -0,0 +1,79 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ExternalCatalogRegistryTest {
+ private static final String ACCOUNT = "000000000000";
+ private ExternalCatalogRegistry registry;
+
+ @BeforeEach
+ @SuppressWarnings("unchecked")
+ void setUp() {
+ StorageFactory factory = mock(StorageFactory.class);
+ when(factory.create(eq("redshift"), any(String.class), any(TypeReference.class)))
+ .thenReturn(new AccountAwareStorageBackend<ExternalSchemaBinding>(new InMemoryStorage<>(), null, ACCOUNT));
+ registry = new ExternalCatalogRegistry(factory);
+ }
+
+ private static ExternalSchemaBinding binding(String cluster, String database, String schema) {
+ return new ExternalSchemaBinding(ACCOUNT, cluster, database, schema, "lake", "arn:aws:iam::000000000000:role/R");
+ }
+
+ @Test
+ void bindsFindsListsAndUnbinds() {
+ registry.bind(binding("c1", "dev", "b"));
+ registry.bind(binding("c1", "dev", "a"));
+
+ assertThat(registry.find(ACCOUNT, "c1", "dev", "a").isPresent(), equalTo(true));
+ assertThat(registry.list(ACCOUNT, "c1", "dev").stream().map(ExternalSchemaBinding::schemaName).toList(),
+ contains("a", "b"));
+ registry.unbind(ACCOUNT, "c1", "dev", "a");
+ assertThat(registry.find(ACCOUNT, "c1", "dev", "a").isPresent(), equalTo(false));
+ }
+
+ @Test
+ void rejectsDuplicateSchema() {
+ registry.bind(binding("c1", "dev", "a"));
+
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,
+ () -> registry.bind(binding("c1", "dev", "a")));
+ assertThat(exception.sqlState(), equalTo("42P06"));
+ }
+
+ @Test
+ void identifiersContainingTheSeparatorDoNotCollideOrLeakAcrossScopes() {
+ registry.bind(binding("c1", "dev|x", "s"));
+ registry.bind(binding("c1", "dev", "x|s"));
+
+ assertThat(registry.list(ACCOUNT, "c1", "dev").stream().map(ExternalSchemaBinding::schemaName).toList(),
+ contains("x|s"));
+ assertThat(registry.list(ACCOUNT, "c1", "dev|x").stream().map(ExternalSchemaBinding::schemaName).toList(),
+ contains("s"));
+ }
+
+ @Test
+ void removeClusterDropsOnlyThatCluster() {
+ registry.bind(binding("c1", "dev", "a"));
+ registry.bind(binding("c10", "dev", "a"));
+
+ registry.removeCluster(ACCOUNT, "c1");
+
+ assertThat(registry.list(ACCOUNT, "c1", "dev"), empty());
+ assertThat(registry.list(ACCOUNT, "c10", "dev").size(), equalTo(1));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriterTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriterTest.java
new file mode 100644
index 0000000000..27c5299e1d
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriterTest.java
@@ -0,0 +1,182 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Column;
+import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import org.junit.jupiter.api.Test;
+
+import java.io.InputStream;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.mock;
+
+class ExternalMetadataWriterTest {
+ private static final ExternalSchemaBinding BINDING = new ExternalSchemaBinding("000000000000",
+ "000000000000:c", "dev", "analytics", "lake", "arn:aws:iam::000000000000:role/R");
+ private final ExternalMetadataWriter writer = new ExternalMetadataWriter(mock(GlueService.class), new ObjectMapper());
+
+ @Test
+ void refreshSqlCallsPrivilegedWriterWithGlueMetadataPayload() {
+ Column id = new Column();
+ id.setName("id");
+ id.setType("int");
+ Column dt = new Column();
+ dt.setName("dt");
+ dt.setType("string");
+ StorageDescriptor descriptor = new StorageDescriptor();
+ descriptor.setLocation("s3://bucket/o'brien/");
+ descriptor.setColumns(List.of(id));
+ Table table = new Table();
+ table.setName("events");
+ table.setStorageDescriptor(descriptor);
+ table.setPartitionKeys(List.of(dt));
+ table.setParameters(Map.of("EXTERNAL", "TRUE"));
+ Partition partition = new Partition();
+ partition.setValues(List.of("2024-01-01"));
+
+ String sql = writer.refreshSql(BINDING, List.of(table), Map.of("events", List.of(partition)));
+
+ assertThat(sql, containsString("SELECT floci_internal.refresh_external_catalog('analytics'"));
+ assertThat(sql, containsString("IAM_ROLE"));
+ assertThat(sql, containsString("\"location\":\"s3://bucket/o''brien/\""));
+ assertThat(sql, containsString("\"columnname\":\"id\",\"external_type\":\"int\""));
+ assertThat(sql, containsString("\"columnname\":\"dt\",\"external_type\":\"string\""));
+ assertThat(sql, containsString("2024-01-01"));
+ assertThat(sql.contains("INSERT INTO floci_internal."), equalTo(false));
+ }
+
+ @Test
+ void refreshSqlUsesPartitionStorageSettings() throws Exception {
+ StorageDescriptor tableDescriptor = new StorageDescriptor();
+ tableDescriptor.setSerdeInfo(serdeInfo("table-serde", Map.of("source", "table")));
+ tableDescriptor.setCompressed(false);
+ Table table = new Table();
+ table.setName("events");
+ table.setStorageDescriptor(tableDescriptor);
+
+ StorageDescriptor partitionDescriptor = new StorageDescriptor();
+ partitionDescriptor.setSerdeInfo(serdeInfo("partition-serde", Map.of("source", "partition")));
+ partitionDescriptor.setCompressed(true);
+ Partition partition = new Partition();
+ partition.setValues(List.of("2024-01-01"));
+ partition.setStorageDescriptor(partitionDescriptor);
+
+ String sql = writer.refreshSql(BINDING, List.of(table), Map.of("events", List.of(partition)));
+ JsonNode partitionRow = refreshPayload(sql).path("partitions").get(0);
+
+ assertEquals("partition-serde", partitionRow.path("serialization_lib").asText());
+ assertEquals("{\"source\":\"partition\"}", partitionRow.path("serde_parameters").asText());
+ assertEquals(1, partitionRow.path("compressed").asInt());
+ }
+
+ @Test
+ void refreshSqlFallsBackToTableSettingsOmittedFromPartitionDescriptor() throws Exception {
+ StorageDescriptor tableDescriptor = new StorageDescriptor();
+ tableDescriptor.setSerdeInfo(serdeInfo("table-serde", Map.of("source", "table")));
+ tableDescriptor.setCompressed(true);
+ Table table = new Table();
+ table.setName("events");
+ table.setStorageDescriptor(tableDescriptor);
+
+ StorageDescriptor partitionDescriptor = new StorageDescriptor();
+ partitionDescriptor.setLocation("s3://bucket/events/date=2024-01-01/");
+ Partition partition = new Partition();
+ partition.setValues(List.of("2024-01-01"));
+ partition.setStorageDescriptor(partitionDescriptor);
+
+ String sql = writer.refreshSql(BINDING, List.of(table), Map.of("events", List.of(partition)));
+ JsonNode partitionRow = refreshPayload(sql).path("partitions").get(0);
+
+ assertEquals("table-serde", partitionRow.path("serialization_lib").asText());
+ assertEquals("{\"source\":\"table\"}", partitionRow.path("serde_parameters").asText());
+ assertEquals(1, partitionRow.path("compressed").asInt());
+ }
+
+ @Test
+ void refreshSqlPreservesExplicitlyEmptyPartitionSerdeParameters() throws Exception {
+ StorageDescriptor tableDescriptor = new StorageDescriptor();
+ tableDescriptor.setSerdeInfo(serdeInfo("table-serde", Map.of("field.delim", "|")));
+ Table table = new Table();
+ table.setName("events");
+ table.setStorageDescriptor(tableDescriptor);
+
+ StorageDescriptor partitionDescriptor = new StorageDescriptor();
+ partitionDescriptor.setSerdeInfo(serdeInfo("partition-serde", Map.of()));
+ Partition partition = new Partition();
+ partition.setValues(List.of("2024-01-01"));
+ partition.setStorageDescriptor(partitionDescriptor);
+
+ String sql = writer.refreshSql(BINDING, List.of(table), Map.of("events", List.of(partition)));
+ JsonNode partitionRow = refreshPayload(sql).path("partitions").get(0);
+
+ assertEquals("{}", partitionRow.path("serde_parameters").asText());
+ }
+
+ @Test
+ void purgeCallsPrivilegedWriterForOneSchema() {
+ String sql = writer.purgeSql("analytics");
+ assertThat(sql, containsString("SELECT floci_internal.purge_external_schema('analytics')"));
+ assertThat(sql.contains("DELETE FROM floci_internal."), equalTo(false));
+ }
+
+ @Test
+ void purgeToleratesMissingBootstrapTables() {
+ BackendSql missingTable = new BackendSql() {
+ @Override
+ public void execute(String sql) {
+ throw new SpectrumReadException("42P01", "metadata table missing");
+ }
+
+ @Override
+ public long copyIn(String copySql, InputStream data) {
+ return 0;
+ }
+ };
+ assertDoesNotThrow(() -> writer.purge(missingTable, "analytics"));
+ }
+
+ @Test
+ void refreshPropagatesMissingBootstrapTables() {
+ BackendSql missingTable = new BackendSql() {
+ @Override
+ public void execute(String sql) {
+ throw new SpectrumReadException("42P01", "metadata table missing");
+ }
+
+ @Override
+ public long copyIn(String copySql, InputStream data) {
+ return 0;
+ }
+ };
+
+ SpectrumReadException exception = assertThrows(SpectrumReadException.class,
+ () -> writer.refresh(missingTable, BINDING.accountId(), BINDING));
+
+ assertEquals("42P01", exception.sqlState());
+ }
+
+ private static StorageDescriptor.SerDeInfo serdeInfo(String library, Map<String, String> parameters) {
+ StorageDescriptor.SerDeInfo serdeInfo = new StorageDescriptor.SerDeInfo();
+ serdeInfo.setSerializationLibrary(library);
+ serdeInfo.setParameters(parameters);
+ return serdeInfo;
+ }
+
+ private JsonNode refreshPayload(String sql) throws Exception {
+ String prefix = "SELECT floci_internal.refresh_external_catalog('analytics', '";
+ String payloadEnd = "'::jsonb)";
+ String escapedJson = sql.substring(prefix.length(), sql.lastIndexOf(payloadEnd));
+ return new ObjectMapper().readTree(escapedJson.replace("''", "'"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScannerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScannerTest.java
new file mode 100644
index 0000000000..df2b92a078
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScannerTest.java
@@ -0,0 +1,186 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalReferenceScanner.Reference;
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+import java.util.Set;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+
+class ExternalReferenceScannerTest {
+ private static final Set<String> SCHEMAS = Set.of("analytics", "Mixed");
+
+ @Test
+ void findsQualifiedReferencesAndIgnoresOtherSchemas() {
+ assertThat(ExternalReferenceScanner.scan("SELECT * FROM analytics.events e JOIN public.users u ON true", SCHEMAS),
+ contains(new Reference("analytics", "events")));
+ }
+
+ @Test
+ void keepsQuotedCaseAndSupportsDatabaseQualifiedNames() {
+ assertThat(ExternalReferenceScanner.scan("SELECT * FROM dev.analytics.events, \"Mixed\".\"Order\"", SCHEMAS),
+ contains(new Reference("analytics", "events"), new Reference("Mixed", "Order")));
+ }
+
+ @Test
+ void findsQualifiedReferencesWithWhitespaceOrCommentsAroundDots() {
+ assertThat(ExternalReferenceScanner.scan(
+ "SELECT * FROM analytics /* schema */ .\n events", SCHEMAS),
+ contains(new Reference("analytics", "events")));
+ }
+
+ @Test
+ void ignoresStringsAndComments() {
+ assertThat(ExternalReferenceScanner.scan("SELECT 'analytics.events' -- analytics.other\n /* analytics.x */", SCHEMAS), empty());
+ }
+
+ @Test
+ void detectsOnlyWritesTargetingExternalTables() {
+ assertThat(ExternalReferenceScanner.writeTarget("INSERT INTO analytics.events VALUES (1)", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ assertThat(ExternalReferenceScanner.writeTarget("INSERT INTO native_copy SELECT * FROM analytics.events", SCHEMAS), equalTo(Optional.empty()));
+ assertThat(ExternalReferenceScanner.writeTarget("WITH values_to_write AS (SELECT 1) UPDATE analytics.events SET id = 2", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ }
+
+ @Test
+ void doesNotTreatColumnQualifiersOrTableAliasesAsTableReferences() {
+ assertThat(ExternalReferenceScanner.scan("SELECT analytics.id FROM public.owners AS analytics", SCHEMAS), empty());
+ assertThat(ExternalReferenceScanner.scan("SELECT * FROM public.owners analytics WHERE analytics.id = 1", SCHEMAS), empty());
+ assertThat(ExternalReferenceScanner.scan(
+ "SELECT * FROM public.owners o JOIN public.teams t ON t.id = analytics.id", SCHEMAS), empty());
+ }
+
+ @Test
+ void findsCommaJoinedAndSubqueryReferencesAndRestoresContextAfterTheSubquery() {
+ assertThat(ExternalReferenceScanner.scan("SELECT * FROM public.a, analytics.events", SCHEMAS),
+ contains(new Reference("analytics", "events")));
+ assertThat(ExternalReferenceScanner.scan(
+ "SELECT * FROM public.t WHERE id IN (SELECT id FROM analytics.events)", SCHEMAS),
+ contains(new Reference("analytics", "events")));
+ assertThat(ExternalReferenceScanner.scan(
+ "SELECT * FROM public.t WHERE id IN (SELECT id FROM public.u) AND analytics.id = 1", SCHEMAS), empty());
+ }
+
+ @Test
+ void checksEveryStatementOfABatchForWrites() {
+ assertThat(ExternalReferenceScanner.writeTarget(
+ "UPDATE public.owners SET x = 1; DELETE FROM analytics.events", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ }
+
+ @Test
+ void detectsWritesInsideDataModifyingCtesTruncateAndMerge() {
+ assertThat(ExternalReferenceScanner.writeTarget(
+ "WITH gone AS (DELETE FROM analytics.events RETURNING id) SELECT * FROM gone", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ assertThat(ExternalReferenceScanner.writeTarget("TRUNCATE TABLE analytics.events", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ assertThat(ExternalReferenceScanner.writeTarget(
+ "TRUNCATE TABLE public.local_table, analytics.events", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ assertThat(ExternalReferenceScanner.writeTarget("MERGE INTO ONLY analytics.events USING public.t ON true", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ assertThat(ExternalReferenceScanner.writeTarget(
+ "MERGE INTO analytics.events USING public.t ON true WHEN MATCHED THEN DELETE", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ }
+
+ @Test
+ void detectsDdlThatWouldChangeAnExternalTableBehindFlocisBack() {
+ Optional<Reference> events = Optional.of(new Reference("analytics", "events"));
+ assertThat(ExternalReferenceScanner.writeTarget("DROP TABLE public.a, analytics.events", SCHEMAS), equalTo(events));
+ assertThat(ExternalReferenceScanner.writeTarget("SELECT 1; DROP TABLE IF EXISTS analytics.events", SCHEMAS), equalTo(events));
+ assertThat(ExternalReferenceScanner.writeTarget("ALTER TABLE analytics.events RENAME TO x", SCHEMAS), equalTo(events));
+ assertThat(ExternalReferenceScanner.writeTarget("ALTER TABLE IF EXISTS ONLY analytics.events ADD COLUMN c int", SCHEMAS), equalTo(events));
+ assertThat(ExternalReferenceScanner.writeTarget("COPY analytics.events FROM 's3://b/k'", SCHEMAS), equalTo(events));
+ assertThat(ExternalReferenceScanner.writeTarget("CREATE TABLE analytics.events (id int)", SCHEMAS), equalTo(events));
+ assertThat(ExternalReferenceScanner.writeTarget("CREATE TEMP TABLE IF NOT EXISTS analytics.events (id int)", SCHEMAS), equalTo(events));
+ }
+
+ @Test
+ void detectsDroppingAnExternalSchemaFromAList() {
+ assertThat(ExternalReferenceScanner.writeTarget("DROP SCHEMA local_schema, analytics CASCADE", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", ""))));
+ }
+
+ @Test
+ void ddlOnNativeObjectsIsNotAnExternalWrite() {
+ assertThat(ExternalReferenceScanner.writeTarget("DROP TABLE public.a, public.b", SCHEMAS), equalTo(Optional.empty()));
+ assertThat(ExternalReferenceScanner.writeTarget("DROP SCHEMA local_schema", SCHEMAS), equalTo(Optional.empty()));
+ assertThat(ExternalReferenceScanner.writeTarget("ALTER TABLE public.t ADD COLUMN c int", SCHEMAS), equalTo(Optional.empty()));
+ assertThat(ExternalReferenceScanner.writeTarget("CREATE TABLE public.t AS SELECT * FROM analytics.events", SCHEMAS), equalTo(Optional.empty()));
+ assertThat(ExternalReferenceScanner.writeTarget("COPY public.t FROM 's3://b/k'", SCHEMAS), equalTo(Optional.empty()));
+ }
+
+ @Test
+ void detectsNativeTablesRenamedOrMovedIntoAnExternalSchema() {
+ assertThat(ExternalReferenceScanner.writeTarget("ALTER TABLE public.t RENAME TO analytics.events", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ assertThat(ExternalReferenceScanner.writeTarget("ALTER TABLE public.t SET SCHEMA analytics", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", ""))));
+ }
+
+ @Test
+ void writesToNativeTablesAreNotExternalWrites() {
+ assertThat(ExternalReferenceScanner.writeTarget("UPDATE public.owners SET x = 1", SCHEMAS), equalTo(Optional.empty()));
+ assertThat(ExternalReferenceScanner.writeTarget(
+ "INSERT INTO public.t VALUES (1) ON CONFLICT DO UPDATE SET x = 1", SCHEMAS), equalTo(Optional.empty()));
+ }
+
+ @Test
+ void writeTargetSeesPastDollarQuotedBodiesAndEscapeStrings() {
+ assertThat(ExternalReferenceScanner.writeTarget(
+ "SELECT $$ it's $$; INSERT INTO analytics.events VALUES (1)", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ assertThat(ExternalReferenceScanner.writeTarget(
+ "SELECT $tag$ it's $$ still inside $tag$; DELETE FROM analytics.events", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ assertThat(ExternalReferenceScanner.writeTarget(
+ "SELECT E'it\\'s'; INSERT INTO analytics.events VALUES (1)", SCHEMAS),
+ equalTo(Optional.of(new Reference("analytics", "events"))));
+ }
+
+ @Test
+ void ignoresReferencesInsideDollarQuotesAndNestedComments() {
+ assertThat(ExternalReferenceScanner.scan("SELECT $$ FROM analytics.events $$", SCHEMAS), empty());
+ assertThat(ExternalReferenceScanner.scan("SELECT 1 /* a /* FROM analytics.events */ FROM analytics.other */", SCHEMAS), empty());
+ }
+
+ @Test
+ void positionalParametersAreNotDollarQuotes() {
+ assertThat(ExternalReferenceScanner.scan("SELECT $1 FROM analytics.events WHERE id = $2", SCHEMAS),
+ contains(new Reference("analytics", "events")));
+ }
+
+ @Test
+ void detectsSelectIntoViewAndIndexTargets() {
+ Optional<Reference> expected = Optional.of(new Reference("analytics", "events"));
+ assertThat(ExternalReferenceScanner.writeTarget("SELECT * INTO analytics.events FROM public.t", SCHEMAS), equalTo(expected));
+ assertThat(ExternalReferenceScanner.writeTarget("CREATE OR REPLACE VIEW analytics.events AS SELECT 1", SCHEMAS), equalTo(expected));
+ assertThat(ExternalReferenceScanner.writeTarget("CREATE MATERIALIZED VIEW analytics.events AS SELECT 1", SCHEMAS), equalTo(expected));
+ assertThat(ExternalReferenceScanner.writeTarget("CREATE UNIQUE INDEX i ON analytics.events (id)", SCHEMAS), equalTo(expected));
+ assertThat(ExternalReferenceScanner.writeTarget("CREATE INDEX i ON public.t (id)", SCHEMAS), equalTo(Optional.empty()));
+ assertThat(ExternalReferenceScanner.writeTarget("SELECT * INTO public.copy FROM analytics.events", SCHEMAS), equalTo(Optional.empty()));
+ }
+
+ @Test
+ void fromInsideExpressionFunctionsIsNotATableList() {
+ assertThat(ExternalReferenceScanner.scan("SELECT EXTRACT(year FROM analytics.ts) FROM public.t", SCHEMAS), empty());
+ assertThat(ExternalReferenceScanner.scan("SELECT SUBSTRING(a FROM analytics.b), TRIM(BOTH 'x' FROM analytics.c) FROM public.t", SCHEMAS), empty());
+ assertThat(ExternalReferenceScanner.scan("SELECT * FROM public.t WHERE a IS DISTINCT FROM analytics.b", SCHEMAS), empty());
+ }
+
+ @Test
+ void tableListInsideExpressionFunctionStillCounts() {
+ assertThat(ExternalReferenceScanner.scan(
+ "SELECT EXTRACT(year FROM (SELECT max(ts) FROM analytics.events)) FROM public.t", SCHEMAS),
+ contains(new Reference("analytics", "events")));
+ assertThat(ExternalReferenceScanner.scan("SELECT EXTRACT(year FROM ts) FROM analytics.events", SCHEMAS),
+ contains(new Reference("analytics", "events")));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParserTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParserTest.java
new file mode 100644
index 0000000000..968a91b265
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParserTest.java
@@ -0,0 +1,242 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class ExternalStatementParserTest {
+ private final ExternalStatementParser parser = new ExternalStatementParser();
+
+ @Test
+ void parsesCreateExternalSchema() {
+ ExternalStatement.CreateSchema schema = (ExternalStatement.CreateSchema) parser.parse(
+ "CREATE EXTERNAL SCHEMA Analytics FROM DATA CATALOG DATABASE 'lake' REGION 'us-east-1' IAM_ROLE 'arn:aws:iam::000000000000:role/R' CREATE EXTERNAL DATABASE IF NOT EXISTS").orElseThrow();
+ assertThat(schema.schemaName(), equalTo("analytics"));
+ assertThat(schema.glueDatabase(), equalTo("lake"));
+ assertThat(schema.createDatabaseIfNotExists(), equalTo(true));
+ }
+
+ @Test
+ void parsesCreateTableWithNestedTypesAndFormat() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (id BIGINT, tags array<string>) STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();
+ assertThat(table.format(), equalTo(ExternalStatement.TableFormat.PARQUET));
+ assertThat(table.columns().get(1).type(), equalTo("array<string>"));
+ }
+
+ @Test
+ void parsesPartitionColumnsWhoseTypesContainParentheses() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (id BIGINT) PARTITIONED BY (price decimal(10,2), day string) "
+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();
+ assertThat(table.partitionColumns().size(), equalTo(2));
+ assertThat(table.partitionColumns().get(0).type(), equalTo("decimal(10,2)"));
+ assertThat(table.partitionColumns().get(1).name(), equalTo("day"));
+ }
+
+ @Test
+ void parsesSerdePropertiesOfAnOpenCsvTable() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (id INT, note VARCHAR(20)) "
+ + "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' "
+ + "WITH SERDEPROPERTIES ('separatorChar'='|', 'quoteChar'='\"') "
+ + "STORED AS TEXTFILE LOCATION 's3://bucket/events/'").orElseThrow();
+ assertThat(table.serde(), equalTo("org.apache.hadoop.hive.serde2.OpenCSVSerde"));
+ assertThat(table.serdeProperties().get("separatorChar"), equalTo("|"));
+ assertThat(table.serdeProperties().get("quoteChar"), equalTo("\""));
+ }
+
+ @Test
+ void parsesSerdePropertyValuesThatContainClosingParentheses() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (id INT) "
+ + "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' "
+ + "WITH SERDEPROPERTIES ('separatorChar'=')', 'quoteChar'='(') "
+ + "STORED AS TEXTFILE LOCATION 's3://bucket/events/'").orElseThrow();
+
+ assertThat(table.serdeProperties().get("separatorChar"), equalTo(")"));
+ assertThat(table.serdeProperties().get("quoteChar"), equalTo("("));
+ }
+
+ @Test
+ void parsesColumnNamesContainingCommasWhenQuoted() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (\"event,id\" VARCHAR, amount DECIMAL(10,2)) "
+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();
+
+ assertThat(table.columns().stream().map(ExternalStatement.ColumnDefinition::name).toList(),
+ equalTo(List.of("event,id", "amount")));
+ }
+
+ @Test
+ void parsesQuotedColumnNamesContainingSpaces() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (\"event name\" VARCHAR(20), amount DECIMAL(10,2)) "
+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();
+
+ assertThat(table.columns().stream().map(ExternalStatement.ColumnDefinition::name).toList(),
+ equalTo(List.of("event name", "amount")));
+ assertThat(table.columns().get(0).type(), equalTo("varchar(20)"));
+ }
+
+ @Test
+ void parsesCreateTableWithCharacterAndVarbyteColumns() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (code CHARACTER(10), payload VARBYTE, memo CHARACTER) "
+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();
+ assertThat(table.columns().get(0).name(), equalTo("code"));
+ assertThat(table.columns().get(0).type(), equalTo("char(10)"));
+ assertThat(table.columns().get(1).name(), equalTo("payload"));
+ assertThat(table.columns().get(1).type(), equalTo("binary"));
+ assertThat(table.columns().get(2).name(), equalTo("memo"));
+ assertThat(table.columns().get(2).type(), equalTo("char(1)"));
+ }
+
+ @Test
+ void defaultsUnsizedCharacterVaryingColumnsToRedshiftLength() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (memo CHARACTER VARYING) "
+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();
+
+ assertThat(table.columns().get(0).type(), equalTo("varchar(256)"));
+ }
+
+ @Test
+ void createDatabaseOptionSurvivesLineBreaksAndExtraWhitespace() {
+ ExternalStatement.CreateSchema schema = (ExternalStatement.CreateSchema) parser.parse(
+ "CREATE EXTERNAL SCHEMA a FROM DATA CATALOG DATABASE 'lake' IAM_ROLE 'arn:aws:iam::000000000000:role/R'\n"
+ + "CREATE EXTERNAL\n DATABASE IF NOT EXISTS").orElseThrow();
+ assertThat(schema.createDatabaseIfNotExists(), equalTo(true));
+ }
+
+ @Test
+ void otherCreateExternalStatementsAreReportedAsUnsupported() {
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,
+ () -> parser.parse("CREATE EXTERNAL FUNCTION f(int) RETURNS int LAMBDA 'x' IAM_ROLE 'r'"));
+ assertThat(error.sqlState(), equalTo("0A000"));
+ assertThat(error.getMessage().contains("unsupported"), equalTo(true));
+ }
+
+ @Test
+ void parsesMultiplePartitionClausesAndPreservesQuotedNames() {
+ ExternalStatement.AddPartitions add = (ExternalStatement.AddPartitions) parser.parse(
+ "ALTER TABLE \"Mixed\".\"Events\" ADD IF NOT EXISTS "
+ + "PARTITION (dt='2024-01-01', region='eu') LOCATION 's3://b/one/' "
+ + "PARTITION (dt='2024-01-02', region='eu') LOCATION 's3://b/two/'").orElseThrow();
+ assertThat(add.schemaName(), equalTo("Mixed"));
+ assertThat(add.tableName(), equalTo("Events"));
+ assertThat(add.ifNotExists(), equalTo(true));
+ assertThat(add.partitions().size(), equalTo(2));
+ assertThat(add.partitions().get(1).values().get("dt"), equalTo("2024-01-02"));
+ }
+
+ @Test
+ void rejectsUnsupportedRoleDefaultAndNonS3Location() {
+ SpectrumSqlException role = assertThrows(SpectrumSqlException.class,
+ () -> parser.parse("CREATE EXTERNAL SCHEMA a FROM DATA CATALOG DATABASE 'd' IAM_ROLE default"));
+ assertThat(role.sqlState(), equalTo("0A000"));
+ SpectrumSqlException location = assertThrows(SpectrumSqlException.class,
+ () -> parser.parse("CREATE EXTERNAL TABLE a.t (id int) STORED AS PARQUET LOCATION 'http://b/x'"));
+ assertThat(location.sqlState(), equalTo("22023"));
+ }
+
+ @Test
+ void rejectsFormatsOutsideTheSupportedAthenaReadPlan() {
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () -> parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (id int) STORED AS ORC LOCATION 's3://bucket/events/'"));
+ assertThat(error.sqlState(), equalTo("0A000"));
+ }
+
+ @Test
+ void dropStatementsDefaultToRestrictAndOnlyCascadeWhenAsked() {
+ assertThat(parser.parse("DROP SCHEMA analytics").orElseThrow(),
+ equalTo(new ExternalStatement.DropSchema("analytics", false, false)));
+ assertThat(parser.parse("DROP SCHEMA analytics RESTRICT").orElseThrow(),
+ equalTo(new ExternalStatement.DropSchema("analytics", false, false)));
+ assertThat(parser.parse("DROP SCHEMA IF EXISTS analytics CASCADE;").orElseThrow(),
+ equalTo(new ExternalStatement.DropSchema("analytics", true, true)));
+ assertThat(parser.parse("DROP TABLE analytics.events").orElseThrow(),
+ equalTo(new ExternalStatement.DropTable("analytics", "events", false, false)));
+ assertThat(parser.parse("DROP TABLE IF EXISTS analytics.events RESTRICT").orElseThrow(),
+ equalTo(new ExternalStatement.DropTable("analytics", "events", true, false)));
+ assertThat(parser.parse("drop table analytics.events cascade").orElseThrow(),
+ equalTo(new ExternalStatement.DropTable("analytics", "events", false, true)));
+ }
+
+ @Test
+ void parsesCreateExternalSchemaIfNotExists() {
+ ExternalStatement.CreateSchema schema = (ExternalStatement.CreateSchema) parser.parse(
+ "CREATE EXTERNAL SCHEMA IF NOT EXISTS Analytics FROM DATA CATALOG DATABASE 'lake' IAM_ROLE 'arn:aws:iam::000000000000:role/R'").orElseThrow();
+ assertThat(schema.schemaName(), equalTo("analytics"));
+ assertThat(schema.ifNotExists(), equalTo(true));
+ ExternalStatement.CreateSchema plain = (ExternalStatement.CreateSchema) parser.parse(
+ "CREATE EXTERNAL SCHEMA analytics FROM DATA CATALOG DATABASE 'lake' IAM_ROLE 'arn:aws:iam::000000000000:role/R'").orElseThrow();
+ assertThat(plain.ifNotExists(), equalTo(false));
+ }
+
+ @Test
+ void canonicalisesRedshiftTypeAliasesToHiveTypes() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (a DOUBLE PRECISION, b int8, c float4, d bool, e character varying(10)) "
+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();
+ assertThat(table.columns().stream().map(ExternalStatement.ColumnDefinition::type).toList(),
+ equalTo(List.of("double", "bigint", "float", "boolean", "varchar(10)")));
+ }
+
+ @Test
+ void parsesTablePropertiesWithParenthesesAndEscapedQuotes() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (id BIGINT) STORED AS PARQUET LOCATION 's3://bucket/events/' "
+ + "TABLE PROPERTIES ('comment'='a (b) it''s', 'numRows'='5')").orElseThrow();
+ assertThat(table.properties().get("comment"), equalTo("a (b) it's"));
+ assertThat(table.properties().get("numRows"), equalTo("5"));
+ }
+
+ @Test
+ void parsesPartitionValuesContainingClosingParentheses() {
+ ExternalStatement.AddPartitions add = (ExternalStatement.AddPartitions) parser.parse(
+ "ALTER TABLE analytics.events ADD PARTITION (region='eu (west)') LOCATION 's3://bucket/events/eu/'").orElseThrow();
+ assertThat(add.partitions().get(0).values().get("region"), equalTo("eu (west)"));
+ assertThat(add.partitions().get(0).location(), equalTo("s3://bucket/events/eu/"));
+ }
+
+ @Test
+ void parsesStatementsThatStartWithComments() {
+ assertThat(parser.parse("-- drop it\n/* now */ DROP TABLE analytics.events").orElseThrow(),
+ equalTo(new ExternalStatement.DropTable("analytics", "events", false, false)));
+ }
+
+ @Test
+ void parsesQuotedNamesContainingDotsAndSpaces() {
+ assertThat(parser.parse("DROP TABLE \"my.schema\".\"my table\"").orElseThrow(),
+ equalTo(new ExternalStatement.DropTable("my.schema", "my table", false, false)));
+ assertThat(parser.parse("DROP SCHEMA \"a b\" CASCADE").orElseThrow(),
+ equalTo(new ExternalStatement.DropSchema("a b", false, true)));
+ }
+
+ @Test
+ void rejectsIamRoleThatIsNotOneQuotedLiteral() {
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class, () -> parser.parse(
+ "CREATE EXTERNAL SCHEMA a FROM DATA CATALOG DATABASE 'lake' IAM_ROLE 'arn'; DROP TABLE x"));
+ assertThat(exception.sqlState(), equalTo("42601"));
+ }
+
+ @Test
+ void reportsMissingStoredAsClearly() {
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class, () -> parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (id BIGINT) LOCATION 's3://bucket/events/'"));
+ assertThat(exception.getMessage(), equalTo("external table requires a STORED AS clause"));
+ }
+
+ @Test
+ void keepsPropertyOrderAsWritten() {
+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(
+ "CREATE EXTERNAL TABLE analytics.events (id BIGINT) STORED AS PARQUET LOCATION 's3://bucket/events/' "
+ + "TABLE PROPERTIES ('z'='1', 'a'='2', 'm'='3')").orElseThrow();
+ assertThat(List.copyOf(table.properties().keySet()), equalTo(List.of("z", "a", "m")));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilderTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilderTest.java
new file mode 100644
index 0000000000..7ad498eb70
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilderTest.java
@@ -0,0 +1,97 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import io.github.hectorvent.floci.services.glue.model.Partition;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatement.ColumnDefinition;
+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatement.CreateTable;
+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatement.PartitionSpec;
+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatement.TableFormat;
+import org.junit.jupiter.api.Test;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class GlueTableBuilderTest {
+ @Test
+ void buildsGlueStorageDescriptorForParquetAndCsv() {
+ CreateTable parquet = new CreateTable("analytics", "events",
+ List.of(new ColumnDefinition("id", "int")), List.of(), TableFormat.PARQUET,
+ "s3://bucket/events/", null, null, Map.of());
+ Table table = GlueTableBuilder.toGlueTable(parquet);
+ assertThat(table.getTableType(), equalTo("EXTERNAL_TABLE"));
+ assertThat(table.getStorageDescriptor().getInputFormat(),
+ equalTo("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"));
+ assertThat(table.getParameters().get("EXTERNAL"), equalTo("TRUE"));
+
+ CreateTable csv = new CreateTable("analytics", "csv_events",
+ List.of(new ColumnDefinition("id", "int")), List.of(), TableFormat.TEXTFILE,
+ "s3://bucket/csv/", "|", null, Map.of());
+ assertThat(GlueTableBuilder.toGlueTable(csv).getStorageDescriptor().getSerdeInfo()
+ .getParameters().get("field.delim"), equalTo("|"));
+ }
+
+ @Test
+ void buildsPartitionValuesInGlueKeyOrder() {
+ CreateTable statement = new CreateTable("analytics", "events",
+ List.of(new ColumnDefinition("id", "int")),
+ List.of(new ColumnDefinition("dt", "string"), new ColumnDefinition("region", "string")),
+ TableFormat.PARQUET, "s3://bucket/events/", null, null, Map.of());
+ Table table = GlueTableBuilder.toGlueTable(statement);
+ Map<String, String> values = new LinkedHashMap<>();
+ values.put("region", "eu");
+ values.put("dt", "2024-01-01");
+
+ Partition partition = GlueTableBuilder.toGluePartition(table,
+ new PartitionSpec(values, "s3://bucket/events/dt=2024-01-01/region=eu/"));
+ assertThat(partition.getValues(), equalTo(List.of("2024-01-01", "eu")));
+ assertThat(partition.getStorageDescriptor().getLocation(),
+ equalTo("s3://bucket/events/dt=2024-01-01/region=eu/"));
+ }
+
+ @Test
+ void openCsvTableKeepsItsSerdePropertiesAndNoDefaultFieldDelimiter() {
+ CreateTable statement = new CreateTable("analytics", "events",
+ List.of(new ColumnDefinition("id", "int")), List.of(), TableFormat.TEXTFILE,
+ "s3://bucket/events/", null, "org.apache.hadoop.hive.serde2.OpenCSVSerde", Map.of(),
+ Map.of("separatorChar", "|", "quoteChar", "\""));
+
+ Map<String, String> parameters = GlueTableBuilder.toGlueTable(statement)
+ .getStorageDescriptor().getSerdeInfo().getParameters();
+
+ assertThat(parameters.get("separatorChar"), equalTo("|"));
+ assertThat(parameters.get("quoteChar"), equalTo("\""));
+ assertThat(parameters.containsKey("field.delim"), equalTo(false));
+ assertThat(GlueTableBuilder.toGlueTable(statement).getStorageDescriptor().getSerdeInfo().getSerializationLibrary(),
+ equalTo("org.apache.hadoop.hive.serde2.OpenCSVSerde"));
+ }
+
+ @Test
+ void rejectsPartitionLocationsOutsideS3AndKeysTheTableDoesNotDeclare() {
+ Table table = GlueTableBuilder.toGlueTable(new CreateTable("analytics", "events", List.of(),
+ List.of(new ColumnDefinition("dt", "string")), TableFormat.PARQUET,
+ "s3://bucket/events/", null, null, Map.of()));
+
+ SpectrumSqlException location = assertThrows(SpectrumSqlException.class, () ->
+ GlueTableBuilder.toGluePartition(table, new PartitionSpec(Map.of("dt", "x"), "/local/path/")));
+ SpectrumSqlException extraKey = assertThrows(SpectrumSqlException.class, () ->
+ GlueTableBuilder.toGluePartition(table, new PartitionSpec(Map.of("dt", "x", "extra", "y"), "s3://b/x/")));
+
+ assertThat(location.sqlState(), equalTo("22023"));
+ assertThat(extraKey.sqlState(), equalTo("22023"));
+ }
+
+ @Test
+ void rejectsMissingPartitionKeys() {
+ Table table = GlueTableBuilder.toGlueTable(new CreateTable("analytics", "events", List.of(),
+ List.of(new ColumnDefinition("dt", "string")), TableFormat.PARQUET,
+ "s3://bucket/events/", null, null, Map.of()));
+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () ->
+ GlueTableBuilder.toGluePartition(table, new PartitionSpec(Map.of("other", "x"), "s3://b/x/")));
+ assertThat(error.sqlState(), equalTo("22023"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapperTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapperTest.java
new file mode 100644
index 0000000000..916b07d884
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapperTest.java
@@ -0,0 +1,65 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import org.junit.jupiter.api.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class GlueTypeMapperTest {
+ @Test
+ void mapsGlueScalarAndParameterizedTypes() {
+ assertThat(GlueTypeMapper.toPostgres("string"), equalTo("text"));
+ assertThat(GlueTypeMapper.toPostgres("int"), equalTo("integer"));
+ assertThat(GlueTypeMapper.toPostgres("DECIMAL(10, 2)"), equalTo("numeric(10,2)"));
+ assertThat(GlueTypeMapper.toPostgres("char(3)"), equalTo("varchar(3)"));
+ assertThat(GlueTypeMapper.toPostgres("character(3)"), equalTo("varchar(3)"));
+ assertThat(GlueTypeMapper.toPostgres("varbyte"), equalTo("bytea"));
+ assertThat(GlueTypeMapper.toPostgres("binary"), equalTo("bytea"));
+ assertThat(GlueTypeMapper.toPostgres("array<string>"), equalTo("jsonb"));
+ assertThat(GlueTypeMapper.isNested("array<string>"), is(true));
+ }
+
+ @Test
+ void rejectsUnknownType() {
+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,
+ () -> GlueTypeMapper.toPostgres("interval"));
+ assertThat(exception.sqlState(), equalTo("0A000"));
+ }
+
+ @Test
+ void quotesProjectionIdentifiersAndPreservesNullMarker() {
+ assertThat(GlueTypeMapper.duckProjection("we\"ird", "string"),
+ equalTo("COALESCE(CAST(\"we\"\"ird\" AS VARCHAR), '\\N') AS \"we\"\"ird\""));
+ assertThat(GlueTypeMapper.duckProjection("payload", "varbyte"),
+ equalTo("COALESCE(CAST('\\x' || hex(\"payload\") AS VARCHAR), '\\N') AS \"payload\""));
+ assertThat(GlueTypeMapper.duckProjection("payload", "binary"),
+ equalTo("COALESCE(CAST('\\x' || hex(\"payload\") AS VARCHAR), '\\N') AS \"payload\""));
+ }
+
+ @Test
+ void canonicalisesRedshiftAliases() {
+ assertThat(GlueTypeMapper.canonicalGlueType("DOUBLE PRECISION"), equalTo("double"));
+ assertThat(GlueTypeMapper.canonicalGlueType("float8"), equalTo("double"));
+ assertThat(GlueTypeMapper.canonicalGlueType("real"), equalTo("float"));
+ assertThat(GlueTypeMapper.canonicalGlueType("int2"), equalTo("smallint"));
+ assertThat(GlueTypeMapper.canonicalGlueType("int4"), equalTo("int"));
+ assertThat(GlueTypeMapper.canonicalGlueType("INT8"), equalTo("bigint"));
+ assertThat(GlueTypeMapper.canonicalGlueType("bool"), equalTo("boolean"));
+ assertThat(GlueTypeMapper.canonicalGlueType("character varying(20)"), equalTo("varchar(20)"));
+ assertThat(GlueTypeMapper.canonicalGlueType("CHARACTER VARYING"), equalTo("varchar(256)"));
+ assertThat(GlueTypeMapper.canonicalGlueType("VARCHAR"), equalTo("varchar(256)"));
+ assertThat(GlueTypeMapper.canonicalGlueType("CHARACTER(10)"), equalTo("char(10)"));
+ assertThat(GlueTypeMapper.canonicalGlueType("character"), equalTo("char(1)"));
+ assertThat(GlueTypeMapper.canonicalGlueType("CHAR"), equalTo("char(1)"));
+ assertThat(GlueTypeMapper.canonicalGlueType("VARBYTE"), equalTo("binary"));
+ assertThat(GlueTypeMapper.canonicalGlueType("varbyte(64000)"), equalTo("binary"));
+ assertThat(GlueTypeMapper.canonicalGlueType("VARBINARY"), equalTo("binary"));
+ assertThat(GlueTypeMapper.canonicalGlueType("binary varying(100)"), equalTo("binary"));
+ assertThat(GlueTypeMapper.canonicalGlueType("Decimal(10, 2)"), equalTo("decimal(10,2)"));
+ assertThat(GlueTypeMapper.canonicalGlueType("NUMERIC(10, 2)"), equalTo("decimal(10,2)"));
+ assertThat(GlueTypeMapper.canonicalGlueType("numeric"), equalTo("decimal"));
+ assertThat(GlueTypeMapper.canonicalGlueType("array<string>"), equalTo("array<string>"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java
new file mode 100644
index 0000000000..c0614bb9d4
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java
@@ -0,0 +1,191 @@
+package io.github.hectorvent.floci.services.redshift.spectrum;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class PostgresBackendSessionTest {
+
+ @Test
+ void copyReadFailureDrainsCopyFailResponseWithoutReplacingOriginalCause() throws Exception {
+ try (ServerSocket listener = new ServerSocket(0);
+ Socket backend = new Socket("localhost", listener.getLocalPort())) {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<Character> copyFailMessage = executor.submit(() -> serveCopyFailure(listener));
+ SpectrumReadException rowFailure = new SpectrumReadException("22000", "bad CSV row");
+ InputStream failingInput = new InputStream() {
+ @Override
+ public int read() throws IOException {
+ throw new IOException("reader failed", rowFailure);
+ }
+ };
+
+ SpectrumReadException thrown = assertThrows(SpectrumReadException.class,
+ () -> new PostgresBackendSession(backend).copyIn("COPY \"t\" FROM STDIN", failingInput));
+
+ assertEquals("58030", thrown.sqlState());
+ assertSame(rowFailure, thrown.getCause().getCause());
+ assertEquals('f', copyFailMessage.get());
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void executeSurfacesBackendErrorWithItsSqlState() throws Exception {
+ try (ServerSocket listener = new ServerSocket(0);
+ Socket backend = new Socket("localhost", listener.getLocalPort())) {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<Character> queryMessage = executor.submit(() -> {
+ try (Socket socket = listener.accept()) {
+ Frame query = readFrame(socket.getInputStream());
+ send(socket.getOutputStream(), 'E', new byte[]{'C', '4', '2', 'P', '0', '1', 0,
+ 'M', 'n', 'o', ' ', 't', 'a', 'b', 'l', 'e', 0, 0});
+ send(socket.getOutputStream(), 'Z', new byte[]{'I'});
+ return query.type();
+ }
+ });
+
+ SpectrumReadException thrown = assertThrows(SpectrumReadException.class,
+ () -> new PostgresBackendSession(backend).execute("SELECT 1"));
+
+ assertEquals("42P01", thrown.sqlState());
+ assertEquals('Q', queryMessage.get());
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void copyInStreamsDataAndReturnsTheRowCountFromTheCommandTag() throws Exception {
+ try (ServerSocket listener = new ServerSocket(0);
+ Socket backend = new Socket("localhost", listener.getLocalPort())) {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<String> received = executor.submit(() -> {
+ try (Socket socket = listener.accept()) {
+ InputStream input = socket.getInputStream();
+ OutputStream output = socket.getOutputStream();
+ readFrame(input);
+ send(output, 'G', new byte[]{0, 0, 0});
+ StringBuilder data = new StringBuilder();
+ Frame frame = readFrame(input);
+ while (frame.type() == 'd') {
+ data.append(new String(frame.body(), StandardCharsets.UTF_8));
+ frame = readFrame(input);
+ }
+ send(output, 'C', "COPY 3\0".getBytes(StandardCharsets.UTF_8));
+ send(output, 'Z', new byte[]{'I'});
+ return data + "|" + frame.type();
+ }
+ });
+
+ long rows = new PostgresBackendSession(backend).copyIn("COPY \"t\" FROM STDIN",
+ new ByteArrayInputStream("a\nb\nc\n".getBytes(StandardCharsets.UTF_8)));
+
+ assertEquals(3, rows);
+ assertEquals("a\nb\nc\n|c", received.get());
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void copyInContinuesAfterAZeroByteRead() throws Exception {
+ try (ServerSocket listener = new ServerSocket(0);
+ Socket backend = new Socket("localhost", listener.getLocalPort())) {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<String> received = executor.submit(() -> {
+ try (Socket socket = listener.accept()) {
+ InputStream input = socket.getInputStream();
+ OutputStream output = socket.getOutputStream();
+ readFrame(input);
+ send(output, 'G', new byte[]{0, 0, 0});
+ StringBuilder data = new StringBuilder();
+ Frame frame = readFrame(input);
+ while (frame.type() == 'd') {
+ data.append(new String(frame.body(), StandardCharsets.UTF_8));
+ frame = readFrame(input);
+ }
+ send(output, 'C', "COPY 1\0".getBytes(StandardCharsets.UTF_8));
+ send(output, 'Z', new byte[]{'I'});
+ return data + "|" + frame.type();
+ }
+ });
+ ByteArrayInputStream source = new ByteArrayInputStream("row\n".getBytes(StandardCharsets.UTF_8));
+ InputStream zeroByteReadInput = new InputStream() {
+ private boolean returnZero = true;
+
+ @Override
+ public int read() {
+ return source.read();
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int length) {
+ if (returnZero) {
+ returnZero = false;
+ return 0;
+ }
+ return source.read(bytes, offset, length);
+ }
+ };
+
+ long rows = new PostgresBackendSession(backend).copyIn("COPY \"t\" FROM STDIN", zeroByteReadInput);
+
+ assertEquals(1, rows);
+ assertEquals("row\n|c", received.get());
+ executor.shutdownNow();
+ }
+ }
+
+
+ private static Character serveCopyFailure(ServerSocket listener) throws IOException {
+ try (Socket socket = listener.accept()) {
+ InputStream input = socket.getInputStream();
+ OutputStream output = socket.getOutputStream();
+ readFrame(input);
+ send(output, 'G', new byte[]{0, 0, 0});
+ Frame copyFail = readFrame(input);
+ byte[] error = new byte[]{'C', '5', '7', '0', '1', '4', 0, 'M', 'C', 'O', 'P', 'Y', ' ', 'f',
+ 'a', 'i', 'l', 'e', 'd', 0, 0};
+ send(output, 'E', error);
+ send(output, 'Z', new byte[]{'I'});
+ return copyFail.type();
+ }
+ }
+
+ private static Frame readFrame(InputStream input) throws IOException {
+ int type = input.read();
+ if (type < 0) {
+ throw new IOException("unexpected EOF");
+ }
+ byte[] length = input.readNBytes(4);
+ int size = ((length[0] & 0xff) << 24) | ((length[1] & 0xff) << 16)
+ | ((length[2] & 0xff) << 8) | (length[3] & 0xff);
+ return new Frame((char) type, input.readNBytes(size - 4));
+ }
+
+ private static void send(OutputStream output, char type, byte[] body) throws IOException {
+ int size = body.length + 4;
+ output.write(type);
+ output.write((size >>> 24) & 0xff);
+ output.write((size >>> 16) & 0xff);
+ output.write((size >>> 8) & 0xff);
+ output.write(size & 0xff);
+ output.write(body);
+ output.flush();
+ }
+
+ private record Frame(char type, byte[] body) {
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java
index 9c633066e7..b2a690d6c0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java
@@ -9,8 +9,8 @@ import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
-import java.util.List;
import java.util.Arrays;
+import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -24,13 +24,17 @@ import static org.mockito.Mockito.when;
class SpectrumMaterializerTest {
@Test
- void materializesRowsWithGeneratedIdentifierAndCopyFraming() throws Exception {
+ void materializesRowsAndSetsSurplusVarcharValuesToNull() throws Exception {
try (ServerSocket listener = new ServerSocket(0); Socket backend = new Socket("localhost", listener.getLocalPort())) {
ExecutorService executor = Executors.newSingleThreadExecutor();
- Future<byte[]> captured = executor.submit(() -> serve(listener));
+ Future<BackendCapture> captured = executor.submit(() -> serve(listener));
SpectrumS3Reader reader = mock(SpectrumS3Reader.class);
when(reader.read(org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.any()))
- .thenReturn(Stream.of(new SpectrumRow(Arrays.asList("1", null)), new SpectrumRow(List.of("two", "x"))));
+ .thenReturn(Stream.of(new SpectrumRow(Arrays.asList("1", null)),
+ new SpectrumRow(List.of("two", "x")),
+ new SpectrumRow(List.of("3", "v".repeat(256))),
+ new SpectrumRow(List.of("4", "v".repeat(257))),
+ new SpectrumRow(List.of("5", "é".repeat(129)))));
SpectrumExternalTable table = new SpectrumExternalTable("000000000000", "dev", "analytics", "events",
List.of(new SpectrumColumn("id", SpectrumColumn.Type.INTEGER),
new SpectrumColumn("value", SpectrumColumn.Type.VARCHAR)),
@@ -40,20 +44,23 @@ class SpectrumMaterializerTest {
.materialize(backend, table, null, reader);
new SpectrumMaterializer().cleanup(backend, materialization);
- byte[] copy = captured.get();
+ BackendCapture capture = captured.get();
assertTrue(materialization.identifier().startsWith("spectrum_tmp_"));
assertFalse(materialization.identifier().contains("events"));
- assertTrue(new String(copy, StandardCharsets.UTF_8).contains("1\t\\N\ntwo\tx\n"));
+ assertTrue(capture.createSql().contains("\"value\" VARCHAR(256)"));
+ String copyRows = new String(capture.copyRows(), StandardCharsets.UTF_8);
+ assertTrue(copyRows.contains("1\t\\N\ntwo\tx\n"));
+ assertTrue(copyRows.contains("3\t" + "v".repeat(256) + "\n4\t\\N\n5\t\\N\n"));
executor.shutdownNow();
}
}
- private static byte[] serve(ServerSocket listener) throws IOException {
+ private static BackendCapture serve(ServerSocket listener) throws IOException {
try (Socket socket = listener.accept()) {
InputStream input = socket.getInputStream();
OutputStream output = socket.getOutputStream();
ByteArrayOutputStream copyRows = new ByteArrayOutputStream();
- readQuery(input);
+ String createSql = readQuery(input);
send(output, 'C', "CREATE 0\0".getBytes(StandardCharsets.UTF_8));
send(output, 'Z', new byte[]{'I'});
readQuery(input);
@@ -72,15 +79,16 @@ class SpectrumMaterializerTest {
readQuery(input);
send(output, 'C', "DROP TABLE\0".getBytes(StandardCharsets.UTF_8));
send(output, 'Z', new byte[]{'I'});
- return copyRows.toByteArray();
+ return new BackendCapture(createSql, copyRows.toByteArray());
}
}
- private static void readQuery(InputStream input) throws IOException {
+ private static String readQuery(InputStream input) throws IOException {
Frame frame = readFrame(input);
if (frame.type() != 'Q') {
throw new IOException("expected query");
}
+ return new String(frame.body(), 0, frame.body().length - 1, StandardCharsets.UTF_8);
}
private static Frame readFrame(InputStream input) throws IOException {
@@ -107,4 +115,7 @@ class SpectrumMaterializerTest {
private record Frame(char type, byte[] body) {
}
+
+ private record BackendCapture(String createSql, byte[] copyRows) {
+ }
}