Light, fluffy, and always free - The AWS Local Emulator alternative
feat(redshift): add external catalog statement parsing and metadata writers (#4679)
Vũ Thế Huyd7d2f73e99c4
diff --git a/docs/services/redshift.md b/docs/services/redshift.mdindex 0092c1b78b..b1508e23c3 100644--- a/docs/services/redshift.md+++ b/docs/services/redshift.md@@ -251,7 +251,8 @@ CREATE EXTERNAL TABLE <schema>.<table> (<name> <type>, ...)```Supported column types are `VARCHAR`, `CHAR`, `INTEGER`, `BIGINT`, `DECIMAL`, `BOOLEAN`, `DATE`,-and `TIMESTAMP`. CSV reads support `DELIMITER`, `QUOTE`, `ESCAPE`, `NULL AS`, and+and `TIMESTAMP`. `CHAR` defaults to length 1 and `VARCHAR` defaults to length 256 when no length+is specified. CSV reads support `DELIMITER`, `QUOTE`, `ESCAPE`, `NULL AS`, and`skip.header.line.count`. Objects under the location are read in lexicographic key order.The query path supports `SELECT *`, explicit column projections, and simple `WHERE` predicatesdiff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.javanew file mode 100644index 0000000000..11e541fc26--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/BackendSql.java@@ -0,0 +1,8 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import java.io.InputStream;++public interface BackendSql {+ void execute(String sql);+ long copyIn(String copySql, InputStream data);+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistry.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistry.javanew file mode 100644index 0000000000..7636b045dc--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistry.java@@ -0,0 +1,80 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.Resettable;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import jakarta.enterprise.context.ApplicationScoped;++import java.net.URLEncoder;+import java.nio.charset.StandardCharsets;+import java.util.Comparator;+import java.util.List;+import java.util.Optional;++@ApplicationScoped+public class ExternalCatalogRegistry implements Resettable {+ private static final char SEPARATOR = '|';+ private final AccountAwareStorageBackend<ExternalSchemaBinding> bindings;++ public ExternalCatalogRegistry(StorageFactory storageFactory) {+ bindings = storageFactory.create("redshift", "redshift-external-schemas.json", new TypeReference<>() { });+ }++ public synchronized void bind(ExternalSchemaBinding binding) {+ String key = key(binding.clusterKey(), binding.databaseName(), binding.schemaName());+ if (bindings.getForAccount(binding.accountId(), key).isPresent()) {+ throw new SpectrumSqlException("42P06", "schema \"" + binding.schemaName() + "\" already exists");+ }+ bindings.putForAccount(binding.accountId(), key, binding);+ bindings.flush();+ }++ public Optional<ExternalSchemaBinding> find(String accountId, String clusterKey, String databaseName, String schemaName) {+ return bindings.getForAccount(accountId, key(clusterKey, databaseName, schemaName));+ }++ public List<ExternalSchemaBinding> list(String accountId, String clusterKey, String databaseName) {+ String prefix = prefix(clusterKey, databaseName);+ return bindings.scanForAccount(accountId, key -> key.startsWith(prefix)).stream()+ .sorted(Comparator.comparing(ExternalSchemaBinding::schemaName)).toList();+ }++ public synchronized void unbind(String accountId, String clusterKey, String databaseName, String schemaName) {+ bindings.deleteForAccount(accountId, key(clusterKey, databaseName, schemaName));+ bindings.flush();+ }++ public synchronized void removeCluster(String accountId, String clusterKey) {+ String prefix = prefix(clusterKey);+ for (String key : bindings.keysForAccount(accountId)) {+ if (key.startsWith(prefix)) {+ bindings.deleteForAccount(accountId, key);+ }+ }+ bindings.flush();+ }++ @Override+ public synchronized void clear() {+ bindings.clear();+ bindings.flush();+ }++ private static String key(String clusterKey, String databaseName, String schemaName) {+ return prefix(clusterKey, databaseName) + encode(schemaName);+ }++ private static String prefix(String clusterKey, String databaseName) {+ return prefix(clusterKey) + encode(databaseName) + SEPARATOR;+ }++ private static String prefix(String clusterKey) {+ return encode(clusterKey) + SEPARATOR;+ }++ /** Percent-encodes a component so a quoted identifier containing the separator cannot collide with another key. */+ private static String encode(String component) {+ return URLEncoder.encode(component, StandardCharsets.UTF_8);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriter.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriter.javanew file mode 100644index 0000000000..fb9c1dd727--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriter.java@@ -0,0 +1,189 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.glue.GlueService;+import io.github.hectorvent.floci.services.glue.model.Column;+import io.github.hectorvent.floci.services.glue.model.Partition;+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;+import io.github.hectorvent.floci.services.glue.model.Table;+import jakarta.enterprise.context.ApplicationScoped;+import org.jboss.logging.Logger;++import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++/** Mirrors Glue metadata into the tables exposed by the SVV external catalog views. */+@ApplicationScoped+public class ExternalMetadataWriter {+ private static final Logger LOG = Logger.getLogger(ExternalMetadataWriter.class);+ private final GlueService glueService;+ private final ObjectMapper objectMapper;++ public ExternalMetadataWriter(GlueService glueService, ObjectMapper objectMapper) {+ this.glueService = glueService;+ this.objectMapper = objectMapper;+ }++ public void refresh(BackendSql backend, String accountId, ExternalSchemaBinding binding) {+ List<Table> tables = RequestScopes.callAs(accountId, () -> glueService.getTables(binding.glueDatabase()));+ Map<String, List<Partition>> partitions = new LinkedHashMap<>();+ for (Table table : tables) {+ partitions.put(table.getName(), RequestScopes.callAs(accountId,+ () -> glueService.getPartitions(binding.glueDatabase(), table.getName())));+ }+ run(backend, refreshSql(binding, tables, partitions), false);+ }++ public void purge(BackendSql backend, String schemaName) {+ run(backend, purgeSql(schemaName), true);+ }++ private void run(BackendSql backend, String sql, boolean tolerateMissingBootstrapTables) {+ try {+ backend.execute(sql);+ } catch (SpectrumReadException exception) {+ if (!tolerateMissingBootstrapTables || !"42P01".equals(exception.sqlState())) {+ throw exception;+ }+ LOG.warnv("Spectrum metadata tables are missing; restart the cluster to re-run the catalog bootstrap: {0}",+ exception.getMessage());+ }+ }++ String purgeSql(String schemaName) {+ return "SELECT floci_internal.purge_external_schema(" + literal(schemaName) + ")";+ }++ String refreshSql(ExternalSchemaBinding binding, List<Table> tables,+ Map<String, List<Partition>> partitionsByTable) {+ List<Map<String, Object>> tableRows = new ArrayList<>();+ List<Map<String, Object>> columnRows = new ArrayList<>();+ List<Map<String, Object>> partitionRows = new ArrayList<>();+ for (Table table : tables) {+ StorageDescriptor descriptor = table.getStorageDescriptor();+ String serdeLibrary = serdeLibrary(descriptor);+ String serdeParameters = json(serdeParameters(descriptor));+ int compressed = descriptor != null && Boolean.TRUE.equals(descriptor.getCompressed()) ? 1 : 0;+ tableRows.add(row(+ "tablename", table.getName(),+ "tabletype", "VIRTUAL_VIEW".equals(table.getTableType()) ? "VIEW" : "TABLE",+ "location", descriptor == null ? "" : descriptor.getLocation(),+ "input_format", descriptor == null ? "" : descriptor.getInputFormat(),+ "output_format", descriptor == null ? "" : descriptor.getOutputFormat(),+ "serialization_lib", serdeLibrary,+ "serde_parameters", serdeParameters,+ "compressed", compressed,+ "parameters", json(table.getParameters())));+ appendColumns(columnRows, table);+ List<Partition> partitions = partitionsByTable.get(table.getName());+ if (partitions != null) {+ for (Partition partition : partitions) {+ appendPartition(partitionRows, table.getName(), partition, descriptor);+ }+ }+ }+ Map<String, Object> payload = row(+ "databasename", binding.glueDatabase(),+ "esoptions", json(Map.of("IAM_ROLE", binding.iamRoleArn())),+ "tables", tableRows,+ "columns", columnRows,+ "partitions", partitionRows);+ return "SELECT floci_internal.refresh_external_catalog(" + literal(binding.schemaName()) + ", "+ + literal(json(payload)) + "::jsonb)";+ }++ private static void appendColumns(List<Map<String, Object>> rows, Table table) {+ StorageDescriptor descriptor = table.getStorageDescriptor();+ List<Column> dataColumns = descriptor == null || descriptor.getColumns() == null+ ? List.of() : descriptor.getColumns();+ List<Column> partitionKeys = table.getPartitionKeys() == null ? List.of() : table.getPartitionKeys();+ int number = 1;+ for (Column column : dataColumns) {+ appendColumn(rows, table.getName(), column, number++, 0);+ }+ int partitionOrder = 1;+ for (Column column : partitionKeys) {+ appendColumn(rows, table.getName(), column, number++, partitionOrder++);+ }+ }++ private static void appendColumn(List<Map<String, Object>> rows, String tableName, Column column,+ int columnNumber, int partitionOrder) {+ rows.add(row("tablename", tableName,+ "columnname", column.getName(),+ "external_type", column.getType(),+ "columnnum", columnNumber,+ "part_key", partitionOrder,+ "is_nullable", "true"));+ }++ private void appendPartition(List<Map<String, Object>> rows, String tableName, Partition partition,+ StorageDescriptor tableDescriptor) {+ StorageDescriptor descriptor = partition.getStorageDescriptor();+ String values = jsonValues(partition.getValues());+ String serializationLibrary = serdeLibrary(descriptor);+ if (serializationLibrary.isBlank()) {+ serializationLibrary = serdeLibrary(tableDescriptor);+ }+ Map<String, String> parameters = serdeParameters(descriptor);+ if (parameters == null) {+ parameters = serdeParameters(tableDescriptor);+ }+ Boolean compressedValue = descriptor == null ? null : descriptor.getCompressed();+ if (compressedValue == null && tableDescriptor != null) {+ compressedValue = tableDescriptor.getCompressed();+ }+ rows.add(row("tablename", tableName,+ "values", values,+ "location", descriptor == null ? "" : descriptor.getLocation(),+ "input_format", descriptor == null ? "" : descriptor.getInputFormat(),+ "output_format", descriptor == null ? "" : descriptor.getOutputFormat(),+ "serialization_lib", serializationLibrary,+ "serde_parameters", json(parameters),+ "compressed", Boolean.TRUE.equals(compressedValue) ? 1 : 0,+ "parameters", json(partition.getParameters())));+ }++ private static String serdeLibrary(StorageDescriptor descriptor) {+ return descriptor == null || descriptor.getSerdeInfo() == null+ || descriptor.getSerdeInfo().getSerializationLibrary() == null+ ? "" : descriptor.getSerdeInfo().getSerializationLibrary();+ }++ private static Map<String, String> serdeParameters(StorageDescriptor descriptor) {+ return descriptor == null || descriptor.getSerdeInfo() == null+ ? null : descriptor.getSerdeInfo().getParameters();+ }++ private String json(Object value) {+ try {+ return objectMapper.writeValueAsString(value == null ? new LinkedHashMap<>() : value);+ } catch (JsonProcessingException exception) {+ throw new IllegalStateException("Unable to serialize external table metadata", exception);+ }+ }++ private String jsonValues(List<String> values) {+ try {+ return objectMapper.writeValueAsString(values == null ? List.of() : values);+ } catch (JsonProcessingException exception) {+ throw new IllegalStateException("Unable to serialize external partition values", exception);+ }+ }++ private static String literal(String value) {+ return "'" + (value == null ? "" : value.replace("'", "''")) + "'";+ }++ private static Map<String, Object> row(Object... values) {+ Map<String, Object> result = new LinkedHashMap<>();+ for (int index = 0; index < values.length; index += 2) {+ result.put((String) values[index], values[index + 1]);+ }+ return result;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScanner.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScanner.javanew file mode 100644index 0000000000..47834bd1e8--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScanner.java@@ -0,0 +1,456 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import java.util.ArrayDeque;+import java.util.ArrayList;+import java.util.Deque;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Locale;+import java.util.Optional;+import java.util.Set;++/**+ * Finds references to bound external schemas while ignoring literals and comments.+ *+ * <p>A qualified name only counts as a table reference where a table can appear: after+ * {@code FROM}, {@code JOIN}, {@code USING}, {@code INTO} or {@code UPDATE}, and in a comma-separated+ * {@code FROM} list. Anywhere else (a select list, {@code ON}, {@code WHERE}) {@code schema.name}+ * is a column qualifier, so a table aliased with an external schema's name is not mistaken for it.+ */+public final class ExternalReferenceScanner {++ /** Keywords after which the following names are table sources. */+ private static final Set<String> TABLE_INTRODUCERS = Set.of("from", "join", "using", "into", "update");++ /** Functions whose argument list uses FROM as an expression separator, not to start a table list. */+ private static final Set<String> EXPRESSION_FROM_FUNCTIONS = Set.of("extract", "substring", "trim", "overlay", "position");++ /** Keywords that end a table-source list: what follows is an expression or a new clause. */+ private static final Set<String> CLAUSE_ENDERS = Set.of("select", "where", "group", "order", "having",+ "limit", "offset", "union", "intersect", "except", "on", "set", "values", "returning", "window",+ "fetch", "qualify");++ public record Reference(String schema, String table) {+ }++ private record Word(String value, boolean quoted) {+ }++ /** An identifier chain ({@code a.b.c}) or one structural character. */+ private record Token(List<Word> chain, char symbol) {++ boolean isChain() {+ return chain != null;+ }++ boolean isKeyword(String keyword) {+ return chain != null && chain.size() == 1 && !chain.getFirst().quoted()+ && chain.getFirst().value().equals(keyword);+ }++ String keyword() {+ return chain != null && chain.size() == 1 && !chain.getFirst().quoted() ? chain.getFirst().value() : null;+ }+ }++ private ExternalReferenceScanner() {+ }++ public static Set<Reference> scan(String sql, Set<String> schemaNames) {+ Set<Reference> result = new LinkedHashSet<>();+ Deque<Boolean> outerContexts = new ArrayDeque<>();+ Deque<Boolean> expressionFromParens = new ArrayDeque<>();+ boolean tableContext = false;+ String previousKeyword = null;+ for (Token token : tokenize(sql)) {+ if (!token.isChain()) {+ switch (token.symbol()) {+ case '(' -> {+ outerContexts.push(tableContext);+ expressionFromParens.push(previousKeyword != null && EXPRESSION_FROM_FUNCTIONS.contains(previousKeyword));+ }+ case ')' -> {+ tableContext = !outerContexts.isEmpty() && outerContexts.pop();+ if (!expressionFromParens.isEmpty()) {+ expressionFromParens.pop();+ }+ }+ case ';' -> {+ tableContext = false;+ outerContexts.clear();+ expressionFromParens.clear();+ }+ default -> {+ // a comma keeps the current context: it separates items of the same list+ }+ }+ previousKeyword = null;+ continue;+ }+ String keyword = token.keyword();+ boolean expressionFrom = "from".equals(keyword)+ && ("distinct".equals(previousKeyword) || !expressionFromParens.isEmpty() && expressionFromParens.peek());+ previousKeyword = keyword;+ if (expressionFrom) {+ continue;+ }+ if (keyword != null && TABLE_INTRODUCERS.contains(keyword)) {+ tableContext = true;+ } else if (keyword != null && CLAUSE_ENDERS.contains(keyword)) {+ tableContext = false;+ } else if (tableContext) {+ Reference reference = reference(token.chain(), schemaNames);+ if (reference != null) {+ result.add(reference);+ }+ }+ }+ return result;+ }++ public static boolean containsIdentifierPrefix(String sql, String prefix) {+ if (sql == null || prefix == null) {+ return false;+ }+ return tokenize(sql).stream().filter(Token::isChain).flatMap(token -> token.chain().stream())+ .anyMatch(word -> word.value().regionMatches(true, 0, prefix, 0, prefix.length()));+ }++ /**+ * The first external table or schema an {@code INSERT}, {@code UPDATE}, {@code DELETE}, {@code MERGE},+ * {@code TRUNCATE}, {@code COPY}, {@code SELECT INTO}, {@code CREATE TABLE}, {@code CREATE VIEW}, {@code CREATE INDEX},+ * {@code ALTER TABLE} or {@code DROP} anywhere in+ * {@code sql} changes: every statement of a batch and every data-modifying CTE is checked, not only+ * the first write. The external DDL Floci implements is parsed before this is reached.+ */+ public static Optional<Reference> writeTarget(String sql, Set<String> schemaNames) {+ if (sql == null) {+ return Optional.empty();+ }+ List<Token> tokens = tokenize(sql);+ for (int i = 0; i < tokens.size(); i++) {+ Optional<Reference> moved = movedIntoExternalSchema(tokens, i, schemaNames);+ if (moved.isPresent()) {+ return moved;+ }+ Optional<Reference> truncated = truncatedTarget(tokens, i, schemaNames);+ if (truncated.isPresent()) {+ return truncated;+ }+ int target = writeTargetIndex(tokens, i);+ if (target >= 0 && target < tokens.size() && tokens.get(target).isChain()) {+ Reference reference = reference(tokens.get(target).chain(), schemaNames);+ if (reference != null) {+ return Optional.of(reference);+ }+ }+ Optional<Reference> dropped = droppedTarget(tokens, i, schemaNames);+ if (dropped.isPresent()) {+ return dropped;+ }+ }+ return Optional.empty();+ }++ private static Optional<Reference> movedIntoExternalSchema(List<Token> tokens, int i, Set<String> schemaNames) {+ if (!keywordAt(tokens, i, "alter") || !keywordAt(tokens, i + 1, "table")) {+ return Optional.empty();+ }+ int source = skipOnly(tokens, i + 2);+ if (keywordAt(tokens, source, "if") && keywordAt(tokens, source + 1, "exists")) {+ source += 2;+ }+ int operation = source + 1;+ if (keywordAt(tokens, operation, "rename") && keywordAt(tokens, operation + 1, "to")) {+ int target = operation + 2;+ if (target < tokens.size() && tokens.get(target).isChain()) {+ return Optional.ofNullable(reference(tokens.get(target).chain(), schemaNames));+ }+ }+ if (keywordAt(tokens, operation, "set") && keywordAt(tokens, operation + 1, "schema")) {+ int target = operation + 2;+ if (target < tokens.size() && tokens.get(target).isChain()) {+ Word schema = tokens.get(target).chain().getFirst();+ if (schemaNames.contains(schema.value())) {+ return Optional.of(new Reference(schema.value(), ""));+ }+ }+ }+ return Optional.empty();+ }++ /** {@code DROP TABLE a, b} and {@code DROP SCHEMA a, b}: any name of the list may be an external one. */+ private static Optional<Reference> droppedTarget(List<Token> tokens, int i, Set<String> schemaNames) {+ if (!keywordAt(tokens, i, "drop")) {+ return Optional.empty();+ }+ boolean table = keywordAt(tokens, i + 1, "table");+ if (!table && !keywordAt(tokens, i + 1, "schema")) {+ return Optional.empty();+ }+ int index = i + 2;+ if (keywordAt(tokens, index, "if") && keywordAt(tokens, index + 1, "exists")) {+ index += 2;+ }+ while (index < tokens.size() && tokens.get(index).isChain()) {+ List<Word> chain = tokens.get(index).chain();+ Reference reference = table ? reference(chain, schemaNames)+ : schemaNames.contains(chain.getFirst().value()) ? new Reference(chain.getFirst().value(), "") : null;+ if (reference != null) {+ return Optional.of(reference);+ }+ boolean anotherName = index + 1 < tokens.size() && !tokens.get(index + 1).isChain()+ && tokens.get(index + 1).symbol() == ',';+ index += anotherName ? 2 : tokens.size();+ }+ return Optional.empty();+ }++ private static int writeTargetIndex(List<Token> tokens, int i) {+ if (keywordAt(tokens, i, "create")) {+ int index = i + 1;+ while (keywordAt(tokens, index, "temp") || keywordAt(tokens, index, "temporary")+ || keywordAt(tokens, index, "unlogged") || keywordAt(tokens, index, "local")) {+ index++;+ }+ if (keywordAt(tokens, index, "or") && keywordAt(tokens, index + 1, "replace")) {+ index += 2;+ while (keywordAt(tokens, index, "temp") || keywordAt(tokens, index, "temporary")) {+ index++;+ }+ }+ if (keywordAt(tokens, index, "materialized")) {+ index++;+ }+ if (keywordAt(tokens, index, "unique")) {+ index++;+ }+ if (keywordAt(tokens, index, "index")) {+ return indexTable(tokens, index + 1);+ }+ if (!(keywordAt(tokens, index, "table") || keywordAt(tokens, index, "view"))) {+ return -1;+ }+ index++;+ if (keywordAt(tokens, index, "if") && keywordAt(tokens, index + 1, "not") && keywordAt(tokens, index + 2, "exists")) {+ index += 3;+ }+ return index;+ }+ if (keywordAt(tokens, i, "alter") && keywordAt(tokens, i + 1, "table")) {+ int index = skipOnly(tokens, i + 2);+ if (keywordAt(tokens, index, "if") && keywordAt(tokens, index + 1, "exists")) {+ index += 2;+ }+ return skipOnly(tokens, index);+ }+ if (keywordAt(tokens, i, "copy")) {+ return i + 1;+ }+ if (keywordAt(tokens, i, "into")) {+ int index = i + 1;+ while (keywordAt(tokens, index, "temp") || keywordAt(tokens, index, "temporary")+ || keywordAt(tokens, index, "unlogged") || keywordAt(tokens, index, "table")) {+ index++;+ }+ return skipOnly(tokens, index);+ }+ if (keywordAt(tokens, i, "update")) {+ return skipOnly(tokens, i + 1);+ }+ if (keywordAt(tokens, i, "delete") && keywordAt(tokens, i + 1, "from")) {+ return skipOnly(tokens, i + 2);+ }+ return -1;+ }++ /** {@code CREATE INDEX [CONCURRENTLY] [IF NOT EXISTS] name ON [ONLY] table}: {@code from} follows INDEX. */+ private static int indexTable(List<Token> tokens, int from) {+ for (int index = from; index < from + 6 && index < tokens.size(); index++) {+ if (keywordAt(tokens, index, "on")) {+ return skipOnly(tokens, index + 1);+ }+ }+ return -1;+ }++ private static Optional<Reference> truncatedTarget(List<Token> tokens, int i, Set<String> schemaNames) {+ if (!keywordAt(tokens, i, "truncate")) {+ return Optional.empty();+ }+ int target = skipOnly(tokens, keywordAt(tokens, i + 1, "table") ? i + 2 : i + 1);+ while (target < tokens.size() && tokens.get(target).isChain()) {+ Reference reference = reference(tokens.get(target).chain(), schemaNames);+ if (reference != null) {+ return Optional.of(reference);+ }+ int comma = target + 1;+ if (comma >= tokens.size() || tokens.get(comma).isChain() || tokens.get(comma).symbol() != ',') {+ return Optional.empty();+ }+ target = skipOnly(tokens, comma + 1);+ }+ return Optional.empty();+ }++ private static int skipOnly(List<Token> tokens, int index) {+ return keywordAt(tokens, index, "only") ? index + 1 : index;+ }++ private static boolean keywordAt(List<Token> tokens, int index, String keyword) {+ return index < tokens.size() && tokens.get(index).isKeyword(keyword);+ }++ private static Reference reference(List<Word> chain, Set<String> schemas) {+ if (chain.size() >= 2 && schemas.contains(chain.get(0).value())) {+ return new Reference(chain.get(0).value(), chain.get(1).value());+ }+ if (chain.size() >= 3 && schemas.contains(chain.get(1).value())) {+ return new Reference(chain.get(1).value(), chain.get(2).value());+ }+ return null;+ }++ private static List<Token> tokenize(String sql) {+ List<Token> result = new ArrayList<>();+ int i = 0;+ while (i < sql.length()) {+ char c = sql.charAt(i);+ if (c == '\'' || c == '-' && i + 1 < sql.length() && sql.charAt(i + 1) == '-'+ || c == '/' && i + 1 < sql.length() && sql.charAt(i + 1) == '*') {+ i = skip(sql, i, c == '\'' && isEscapeStringPrefix(sql, i));+ } else if (c == '$' && dollarTagEnd(sql, i) > 0) {+ i = skipDollarQuoted(sql, i);+ } else if (c == '"' || Character.isLetter(c) || c == '_') {+ List<Word> chain = new ArrayList<>();+ i = read(sql, i, chain);+ while (true) {+ int separator = skipTrivia(sql, i);+ if (separator >= sql.length() || sql.charAt(separator) != '.') {+ break;+ }+ int next = skipTrivia(sql, separator + 1);+ if (next >= sql.length() || !(sql.charAt(next) == '"' || Character.isLetter(sql.charAt(next))+ || sql.charAt(next) == '_')) {+ break;+ }+ i = read(sql, next, chain);+ }+ result.add(new Token(chain, '\0'));+ } else {+ if (c == '(' || c == ')' || c == ',' || c == ';') {+ result.add(new Token(null, c));+ }+ i++;+ }+ }+ return result;+ }++ private static int skipTrivia(String sql, int start) {+ int i = start;+ while (i < sql.length()) {+ if (Character.isWhitespace(sql.charAt(i))) {+ i++;+ } else if (sql.charAt(i) == '-' && i + 1 < sql.length() && sql.charAt(i + 1) == '-'+ || sql.charAt(i) == '/' && i + 1 < sql.length() && sql.charAt(i + 1) == '*') {+ i = skip(sql, i, false);+ } else {+ break;+ }+ }+ return i;+ }++ private static int read(String sql, int start, List<Word> chain) {+ if (sql.charAt(start) == '"') {+ StringBuilder value = new StringBuilder();+ int i = start + 1;+ while (i < sql.length()) {+ if (sql.charAt(i) == '"') {+ if (i + 1 < sql.length() && sql.charAt(i + 1) == '"') {+ value.append('"');+ i += 2;+ } else {+ i++;+ break;+ }+ } else {+ value.append(sql.charAt(i++));+ }+ }+ chain.add(new Word(value.toString(), true));+ return i;+ }+ int i = start;+ while (i < sql.length() && (Character.isLetterOrDigit(sql.charAt(i)) || sql.charAt(i) == '_'+ || sql.charAt(i) == '$')) {+ i++;+ }+ chain.add(new Word(sql.substring(start, i).toLowerCase(Locale.ROOT), false));+ return i;+ }++ private static boolean isEscapeStringPrefix(String sql, int quote) {+ if (quote == 0 || Character.toLowerCase(sql.charAt(quote - 1)) != 'e') {+ return false;+ }+ if (quote == 1) {+ return true;+ }+ char before = sql.charAt(quote - 2);+ return !(Character.isLetterOrDigit(before) || before == '_' || before == '$');+ }++ /** Returns the index just past an opening dollar-quote tag such as {@code $$} or {@code $body$}, or -1. */+ private static int dollarTagEnd(String sql, int start) {+ int i = start + 1;+ if (i < sql.length() && (Character.isLetter(sql.charAt(i)) || sql.charAt(i) == '_')) {+ while (i < sql.length() && (Character.isLetterOrDigit(sql.charAt(i)) || sql.charAt(i) == '_')) {+ i++;+ }+ }+ return i < sql.length() && sql.charAt(i) == '$' ? i + 1 : -1;+ }++ private static int skipDollarQuoted(String sql, int start) {+ int bodyStart = dollarTagEnd(sql, start);+ int close = sql.indexOf(sql.substring(start, bodyStart), bodyStart);+ return close < 0 ? sql.length() : close + (bodyStart - start);+ }++ private static int skip(String sql, int start, boolean backslashEscapes) {+ if (sql.charAt(start) == '\'') {+ int i = start + 1;+ while (i < sql.length()) {+ char c = sql.charAt(i);+ if (backslashEscapes && c == '\\') {+ i += 2;+ } else if (c == '\'' && (i + 1 >= sql.length() || sql.charAt(i + 1) != '\'')) {+ return i + 1;+ } else {+ i += c == '\'' ? 2 : 1;+ }+ }+ return sql.length();+ }+ if (sql.charAt(start) == '-') {+ int i = sql.indexOf('\n', start + 2);+ return i < 0 ? sql.length() : i;+ }+ int depth = 1;+ int i = start + 2;+ while (i < sql.length() && depth > 0) {+ if (sql.startsWith("/*", i)) {+ depth++;+ i += 2;+ } else if (sql.startsWith("*/", i)) {+ depth--;+ i += 2;+ } else {+ i++;+ }+ }+ return Math.min(i, sql.length());+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalSchemaBinding.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalSchemaBinding.javanew file mode 100644index 0000000000..d935dab68c--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalSchemaBinding.java@@ -0,0 +1,22 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+public record ExternalSchemaBinding(String accountId, String clusterKey, String databaseName,+ String schemaName, String glueDatabase, String iamRoleArn) {+ public ExternalSchemaBinding {+ require(accountId, "accountId");+ require(clusterKey, "clusterKey");+ require(databaseName, "databaseName");+ require(schemaName, "schemaName");+ require(glueDatabase, "glueDatabase");+ require(iamRoleArn, "iamRoleArn");+ }++ private static void require(String value, String field) {+ if (value == null || value.isBlank()) {+ throw new IllegalArgumentException(field + " is required");+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatement.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatement.javanew file mode 100644index 0000000000..096ccfe818--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatement.java@@ -0,0 +1,46 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import java.util.Collections;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++public sealed interface ExternalStatement permits ExternalStatement.CreateSchema, ExternalStatement.CreateTable,+ ExternalStatement.AddPartitions, ExternalStatement.DropSchema, ExternalStatement.DropTable {+ /** Read-only copy that keeps statement order, so Glue payloads are stable across runs. */+ private static <V> Map<String, V> ordered(Map<String, V> source) {+ return Collections.unmodifiableMap(new LinkedHashMap<>(source));+ }++ record ColumnDefinition(String name, String type) { }+ enum TableFormat { PARQUET, TEXTFILE, JSON }+ record PartitionSpec(Map<String, String> values, String location) {+ public PartitionSpec { values = ordered(values); }+ }+ record CreateSchema(String schemaName, String glueDatabase, String iamRoleArn, boolean createDatabaseIfNotExists,+ boolean ifNotExists) implements ExternalStatement { }+ record CreateTable(String schemaName, String tableName, List<ColumnDefinition> columns,+ List<ColumnDefinition> partitionColumns, TableFormat format, String location,+ String delimiter, String serde, Map<String, String> properties,+ Map<String, String> serdeProperties) implements ExternalStatement {+ public CreateTable {+ columns = List.copyOf(columns);+ partitionColumns = List.copyOf(partitionColumns);+ properties = ordered(properties);+ serdeProperties = ordered(serdeProperties);+ }++ public CreateTable(String schemaName, String tableName, List<ColumnDefinition> columns,+ List<ColumnDefinition> partitionColumns, TableFormat format, String location,+ String delimiter, String serde, Map<String, String> properties) {+ this(schemaName, tableName, columns, partitionColumns, format, location, delimiter, serde, properties, Map.of());+ }+ }+ record AddPartitions(String schemaName, String tableName, boolean ifNotExists,+ List<PartitionSpec> partitions) implements ExternalStatement {+ public AddPartitions { partitions = List.copyOf(partitions); }+ }+ /** {@code cascade} is true only when the statement said CASCADE; RESTRICT is the default, as in PostgreSQL and Redshift. */+ record DropSchema(String schemaName, boolean ifExists, boolean cascade) implements ExternalStatement { }+ record DropTable(String schemaName, String tableName, boolean ifExists, boolean cascade) implements ExternalStatement { }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParser.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParser.javanew file mode 100644index 0000000000..a4a41bf237--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParser.java@@ -0,0 +1,323 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import jakarta.enterprise.context.ApplicationScoped;++import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.Optional;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++@ApplicationScoped+public class ExternalStatementParser {+ private static final Pattern CREATE_SCHEMA = Pattern.compile("(?is)^\\s*CREATE\\s+EXTERNAL\\s+SCHEMA\\s+(IF\\s+NOT\\s+EXISTS\\s+)?(.+?)\\s+FROM\\s+DATA\\s+CATALOG\\s+DATABASE\\s+'((?:''|[^'])*)'(?:\\s+REGION\\s+'(?:''|[^'])*')?\\s+IAM_ROLE\\s+(.+?)(\\s+CREATE\\s+EXTERNAL\\s+DATABASE\\s+IF\\s+NOT\\s+EXISTS)?\\s*;?\\s*$");+ private static final Pattern CREATE_EXTERNAL_TABLE = Pattern.compile("(?is)^\\s*CREATE\\s+EXTERNAL\\s+TABLE\\b.*");+ private static final Pattern CREATE_EXTERNAL = Pattern.compile("(?is)^\\s*CREATE\\s+EXTERNAL\\b.*");+ private static final Pattern SERDE_PROPERTIES = Pattern.compile("(?is)WITH\\s+SERDEPROPERTIES\\s*\\(");+ private static final String NAME = "(\"(?:\"\"|[^\"])+\"|[^.;,\\s\"()]+)";+ private static final Pattern CREATE_TABLE_HEAD = Pattern.compile("(?is)^\\s*CREATE\\s+EXTERNAL\\s+TABLE\\s+" + NAME + "\\s*\\.\\s*" + NAME + "\\s*\\(");+ private static final Pattern DROP_TABLE = Pattern.compile("(?is)^\\s*DROP\\s+TABLE\\s+(IF\\s+EXISTS\\s+)?" + NAME + "\\.\\s*" + NAME + "(?:\\s+(CASCADE|RESTRICT))?\\s*;?\\s*$");+ private static final Pattern DROP_SCHEMA = Pattern.compile("(?is)^\\s*DROP\\s+SCHEMA\\s+(IF\\s+EXISTS\\s+)?" + NAME + "(?:\\s+(CASCADE|RESTRICT))?\\s*;?\\s*$");+ private static final Pattern TABLE_PROPERTIES = Pattern.compile("(?is)(?:TBLPROPERTIES|TABLE\\s+PROPERTIES)\\s*\\(");+ private static final Pattern PARTITION_VALUE = Pattern.compile("(?is)([^,=]+)=\\s*'((?:''|[^'])*)'\\s*(?:,|$)");+ private static final Pattern LOCATION_VALUE = Pattern.compile("(?is)LOCATION\\s+'((?:''|[^'])*)'");+ private static final Pattern SERDE_VALUE = Pattern.compile("(?is)SERDE\\s+'((?:''|[^'])*)'");+ private static final Pattern TERMINATED_BY_VALUE = Pattern.compile("(?is)TERMINATED BY\\s+'((?:''|[^'])*)'");+ private static final Pattern STORED_AS_VALUE = Pattern.compile("(?is)STORED AS\\s+([A-Za-z]+)");+ private static final Pattern QUOTED_LITERAL = Pattern.compile("'(?:''|[^'])*'");+ private static final Pattern QUOTED_PAIR = Pattern.compile("'((?:''|[^'])*)'\\s*=\\s*'((?:''|[^'])*)'");+ private static final Pattern PARTITION_HEAD = Pattern.compile("(?is)PARTITION\\s*\\(");+ private static final Pattern PARTITION_LOCATION = Pattern.compile("(?is)\\s*LOCATION\\s+'((?:''|[^'])*)'");+ private static final Pattern PARTITIONED_BY = Pattern.compile("(?is)PARTITIONED\\s+BY\\s*\\(");+ private static final Pattern ADD_PARTITIONS = Pattern.compile("(?is)^\\s*ALTER\\s+TABLE\\s+" + NAME + "\\.\\s*" + NAME + "\\s+ADD\\s+(IF\\s+NOT\\s+EXISTS\\s+)?(.+?)\\s*;?\\s*$");++ public Optional<ExternalStatement> parse(String sql) {+ if (sql == null) {+ return Optional.empty();+ }+ String text = stripLeadingComments(sql);+ Matcher schema = CREATE_SCHEMA.matcher(text);+ if (schema.matches()) {+ String role = schema.group(4).trim();+ if (role.equalsIgnoreCase("default")) {+ throw new SpectrumSqlException("0A000", "IAM_ROLE DEFAULT is not supported");+ }+ if (!QUOTED_LITERAL.matcher(role).matches()) {+ throw new SpectrumSqlException("42601", "IAM_ROLE must be a single quoted role ARN");+ }+ return Optional.of(new ExternalStatement.CreateSchema(identifier(schema.group(2)),+ schema.group(3).replace("''", "'"), unquote(role), schema.group(5) != null,+ schema.group(1) != null));+ }+ if (CREATE_EXTERNAL_TABLE.matcher(text).matches()) {+ return Optional.of(parseTable(text));+ }+ if (CREATE_EXTERNAL.matcher(text).matches()) {+ throw new SpectrumSqlException("0A000", "unsupported CREATE EXTERNAL statement");+ }+ Matcher dropTable = DROP_TABLE.matcher(text);+ if (dropTable.matches()) {+ return Optional.of(new ExternalStatement.DropTable(identifier(dropTable.group(2)), identifier(dropTable.group(3)),+ dropTable.group(1) != null, "CASCADE".equalsIgnoreCase(dropTable.group(4))));+ }+ Matcher dropSchema = DROP_SCHEMA.matcher(text);+ if (dropSchema.matches()) {+ return Optional.of(new ExternalStatement.DropSchema(identifier(dropSchema.group(2)),+ dropSchema.group(1) != null, "CASCADE".equalsIgnoreCase(dropSchema.group(3))));+ }+ Matcher addPartitions = ADD_PARTITIONS.matcher(text);+ if (addPartitions.matches()) {+ List<ExternalStatement.PartitionSpec> partitions = parsePartitions(addPartitions.group(4));+ return partitions.isEmpty() ? Optional.empty() : Optional.of(new ExternalStatement.AddPartitions(+ identifier(addPartitions.group(1)), identifier(addPartitions.group(2)),+ addPartitions.group(3) != null, partitions));+ }+ return Optional.empty();+ }++ private ExternalStatement.CreateTable parseTable(String text) {+ Matcher matcher = CREATE_TABLE_HEAD.matcher(text);+ if (!matcher.find()) {+ throw new SpectrumSqlException("0A000", "malformed CREATE EXTERNAL TABLE");+ }+ int open = matcher.end() - 1;+ int close = matchingParen(text, open);+ if (close < 0) {+ throw new SpectrumSqlException("0A000", "malformed CREATE EXTERNAL TABLE column list");+ }+ String tail = text.substring(close + 1);+ List<ExternalStatement.ColumnDefinition> columns = columns(text.substring(open + 1, close));+ List<ExternalStatement.ColumnDefinition> partitions = List.of();+ Matcher partition = PARTITIONED_BY.matcher(tail);+ if (partition.find()) {+ // Partition columns may have types such as decimal(10,2), so match nested parentheses.+ int partitionOpen = partition.end() - 1;+ int partitionClose = matchingParen(tail, partitionOpen);+ if (partitionClose < 0) {+ throw new SpectrumSqlException("0A000", "malformed CREATE EXTERNAL TABLE partition list");+ }+ partitions = columns(tail.substring(partitionOpen + 1, partitionClose));+ }+ String location = value(tail, LOCATION_VALUE);+ if (location == null || !location.startsWith("s3://")) {+ throw new SpectrumSqlException("22023", "external table location must be an s3:// URI");+ }+ String serde = value(tail, SERDE_VALUE);+ String delimiter = value(tail, TERMINATED_BY_VALUE);+ String stored = keywordValue(tail, STORED_AS_VALUE);+ if (stored == null) {+ throw new SpectrumSqlException("0A000", "external table requires a STORED AS clause");+ }+ if (!(stored.equalsIgnoreCase("PARQUET") || stored.equalsIgnoreCase("TEXTFILE"))) {+ throw new SpectrumSqlException("0A000", "unsupported external table format: " + stored);+ }+ ExternalStatement.TableFormat format = stored.equalsIgnoreCase("PARQUET")+ ? ExternalStatement.TableFormat.PARQUET+ : serde != null && serde.toLowerCase(Locale.ROOT).contains("json")+ ? ExternalStatement.TableFormat.JSON : ExternalStatement.TableFormat.TEXTFILE;+ Map<String, String> properties = properties(tail);+ return new ExternalStatement.CreateTable(identifier(matcher.group(1)), identifier(matcher.group(2)), columns, partitions,+ format, location, delimiter, serde, properties, serdeProperties(tail));+ }++ private static String stripLeadingComments(String sql) {+ String text = sql.trim();+ while (true) {+ int end;+ if (text.startsWith("--")) {+ end = text.indexOf('\n');+ end = end < 0 ? text.length() : end;+ } else if (text.startsWith("/*")) {+ end = text.indexOf("*/", 2);+ end = end < 0 ? text.length() : end + 2;+ } else {+ return text;+ }+ text = text.substring(end).trim();+ }+ }++ private static int matchingParen(String text, int open) {+ int depth = 0;+ char quote = 0;+ for (int i = open; i < text.length(); i++) {+ char c = text.charAt(i);+ if (quote != 0) {+ if (c == quote) {+ if (i + 1 < text.length() && text.charAt(i + 1) == quote) {+ i++;+ } else {+ quote = 0;+ }+ }+ } else if (c == '\'' || c == '"') {+ quote = c;+ } else if (c == '(') {+ depth++;+ } else if (c == ')' && --depth == 0) {+ return i;+ }+ }+ return -1;+ }++ private static List<ExternalStatement.ColumnDefinition> columns(String text) {+ List<ExternalStatement.ColumnDefinition> result = new ArrayList<>();+ for (String item : split(text)) {+ String definition = item.trim();+ int nameEnd = columnNameEnd(definition);+ if (nameEnd < 0 || nameEnd == definition.length() || definition.substring(nameEnd).isBlank()) {+ throw new SpectrumSqlException("0A000", "malformed column definition");+ }+ result.add(new ExternalStatement.ColumnDefinition(identifier(definition.substring(0, nameEnd)),+ GlueTypeMapper.canonicalGlueType(definition.substring(nameEnd).trim())));+ }+ return result;+ }++ private static int columnNameEnd(String definition) {+ if (!definition.startsWith("\"")) {+ for (int index = 0; index < definition.length(); index++) {+ if (Character.isWhitespace(definition.charAt(index))) {+ return index;+ }+ }+ return -1;+ }+ for (int index = 1; index < definition.length(); index++) {+ if (definition.charAt(index) == '"') {+ if (index + 1 < definition.length() && definition.charAt(index + 1) == '"') {+ index++;+ } else {+ return index + 1;+ }+ }+ }+ return -1;+ }++ private static List<String> split(String text) {+ List<String> result = new ArrayList<>();+ int depth = 0;+ int start = 0;+ char quote = 0;+ for (int i = 0; i < text.length(); i++) {+ char c = text.charAt(i);+ if (quote != 0) {+ if (c == quote) {+ if (i + 1 < text.length() && text.charAt(i + 1) == quote) {+ i++;+ } else {+ quote = 0;+ }+ }+ } else if (c == '\'' || c == '"') {+ quote = c;+ } else if (c == '(' || c == '<') {+ depth++;+ } else if (c == ')' || c == '>') {+ depth--;+ } else if (c == ',' && depth == 0) {+ result.add(text.substring(start, i));+ start = i + 1;+ }+ }+ result.add(text.substring(start));+ return result;+ }++ private static String value(String text, Pattern pattern) {+ Matcher matcher = pattern.matcher(text);+ return matcher.find() ? matcher.group(1).replace("''", "'") : null;+ }++ private static String keywordValue(String text, Pattern pattern) {+ Matcher matcher = pattern.matcher(text);+ return matcher.find() ? matcher.group(1) : null;+ }++ private static Map<String, String> serdeProperties(String text) {+ Map<String, String> result = new LinkedHashMap<>();+ Matcher matcher = SERDE_PROPERTIES.matcher(text);+ if (matcher.find()) {+ int open = matcher.end() - 1;+ int close = matchingParen(text, open);+ if (close < 0) {+ throw new SpectrumSqlException("0A000", "malformed SERDEPROPERTIES clause");+ }+ result.putAll(quotedPairs(text.substring(open + 1, close)));+ }+ return result;+ }++ private static Map<String, String> properties(String text) {+ Map<String, String> result = new LinkedHashMap<>();+ Matcher matcher = TABLE_PROPERTIES.matcher(text);+ if (matcher.find()) {+ int open = matcher.end() - 1;+ int close = matchingParen(text, open);+ if (close < 0) {+ throw new SpectrumSqlException("0A000", "malformed TABLE PROPERTIES clause");+ }+ result.putAll(quotedPairs(text.substring(open + 1, close)));+ }+ return result;+ }++ private static Map<String, String> quotedPairs(String text) {+ Map<String, String> result = new LinkedHashMap<>();+ Matcher pair = QUOTED_PAIR.matcher(text);+ while (pair.find()) {+ result.put(pair.group(1).replace("''", "'"), pair.group(2).replace("''", "'"));+ }+ return result;+ }++ private static List<ExternalStatement.PartitionSpec> parsePartitions(String text) {+ List<ExternalStatement.PartitionSpec> result = new ArrayList<>();+ Matcher head = PARTITION_HEAD.matcher(text);+ int end = 0;+ while (head.find(end)) {+ if (!text.substring(end, head.start()).isBlank()) {+ return List.of();+ }+ int open = head.end() - 1;+ int close = matchingParen(text, open);+ if (close < 0) {+ return List.of();+ }+ Matcher location = PARTITION_LOCATION.matcher(text).region(close + 1, text.length());+ if (!location.lookingAt()) {+ return List.of();+ }+ Map<String, String> values = new LinkedHashMap<>();+ Matcher value = PARTITION_VALUE.matcher(text.substring(open + 1, close).trim() + ",");+ while (value.find()) {+ values.put(identifier(value.group(1)), value.group(2).replace("''", "'"));+ }+ if (values.isEmpty()) {+ return List.of();+ }+ result.add(new ExternalStatement.PartitionSpec(values, location.group(1).replace("''", "'")));+ end = location.end();+ }+ return end == text.length() || text.substring(end).isBlank() ? result : List.of();+ }++ private static String identifier(String value) {+ String trimmed = value.trim();+ if (trimmed.startsWith("\"") && trimmed.endsWith("\"")) {+ return trimmed.substring(1, trimmed.length() - 1).replace("\"\"", "\"");+ }+ return trimmed.toLowerCase(Locale.ROOT);+ }++ private static String unquote(String value) {+ String trimmed = value.trim();+ return trimmed.startsWith("'") && trimmed.endsWith("'")+ ? trimmed.substring(1, trimmed.length() - 1).replace("''", "'") : trimmed;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilder.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilder.javanew file mode 100644index 0000000000..d162c4bee5--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilder.java@@ -0,0 +1,115 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import io.github.hectorvent.floci.services.glue.model.Column;+import io.github.hectorvent.floci.services.glue.model.Partition;+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;+import io.github.hectorvent.floci.services.glue.model.Table;++import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Locale;+import java.util.Map;++public final class GlueTableBuilder {+ private static final String TEXT_INPUT_FORMAT = "org.apache.hadoop.mapred.TextInputFormat";+ private static final String TEXT_OUTPUT_FORMAT = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat";+ private static final String LAZY_SIMPLE_SERDE = "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";+ private static final String OPEN_CSV_SERDE = "org.apache.hadoop.hive.serde2.OpenCSVSerde";++ private GlueTableBuilder() {+ }++ public static Table toGlueTable(ExternalStatement.CreateTable statement) {+ StorageDescriptor descriptor = new StorageDescriptor();+ descriptor.setLocation(statement.location());+ descriptor.setColumns(columns(statement.columns()));+ StorageDescriptor.SerDeInfo serde = new StorageDescriptor.SerDeInfo();+ Map<String, String> serdeParameters = new LinkedHashMap<>();+ switch (statement.format()) {+ case PARQUET -> {+ descriptor.setInputFormat("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat");+ descriptor.setOutputFormat("org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat");+ serde.setSerializationLibrary("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe");+ }+ case JSON -> {+ descriptor.setInputFormat(TEXT_INPUT_FORMAT);+ descriptor.setOutputFormat(TEXT_OUTPUT_FORMAT);+ serde.setSerializationLibrary(statement.serde() == null ? "org.openx.data.jsonserde.JsonSerDe" : statement.serde());+ }+ case TEXTFILE -> {+ descriptor.setInputFormat(TEXT_INPUT_FORMAT);+ descriptor.setOutputFormat(TEXT_OUTPUT_FORMAT);+ if (isOpenCsv(statement.serde())) {+ // OpenCSVSerde takes its delimiter from separatorChar, which the SERDEPROPERTIES supply+ serde.setSerializationLibrary(OPEN_CSV_SERDE);+ } else {+ serde.setSerializationLibrary(LAZY_SIMPLE_SERDE);+ String delimiter = statement.delimiter() == null ? "," : statement.delimiter();+ serdeParameters.put("field.delim", delimiter);+ serdeParameters.put("serialization.format", delimiter);+ }+ }+ }+ serdeParameters.putAll(statement.serdeProperties());+ serde.setParameters(serdeParameters);+ descriptor.setSerdeInfo(serde);+ Table table = new Table();+ table.setName(statement.tableName());+ table.setTableType("EXTERNAL_TABLE");+ table.setStorageDescriptor(descriptor);+ table.setPartitionKeys(columns(statement.partitionColumns()));+ Map<String, String> parameters = new LinkedHashMap<>(statement.properties());+ parameters.put("EXTERNAL", "TRUE");+ table.setParameters(parameters);+ return table;+ }++ public static Partition toGluePartition(Table table, ExternalStatement.PartitionSpec spec) {+ if (spec.location() == null || !spec.location().startsWith("s3://")) {+ throw new SpectrumSqlException("22023", "partition location must be an s3:// URI");+ }+ List<String> values = new ArrayList<>();+ for (Column key : table.getPartitionKeys()) {+ String value = spec.values().entrySet().stream()+ .filter(entry -> entry.getKey().equalsIgnoreCase(key.getName()))+ .map(Map.Entry::getValue).findFirst().orElse(null);+ if (value == null) {+ throw new SpectrumSqlException("22023", "partition key \"" + key.getName() + "\" is missing");+ }+ values.add(value);+ }+ for (String name : spec.values().keySet()) {+ boolean declared = table.getPartitionKeys().stream().anyMatch(key -> key.getName().equalsIgnoreCase(name));+ if (!declared) {+ throw new SpectrumSqlException("22023", "\"" + name + "\" is not a partition key of the table");+ }+ }+ StorageDescriptor source = table.getStorageDescriptor();+ StorageDescriptor descriptor = new StorageDescriptor();+ descriptor.setColumns(source.getColumns());+ descriptor.setInputFormat(source.getInputFormat());+ descriptor.setOutputFormat(source.getOutputFormat());+ descriptor.setSerdeInfo(source.getSerdeInfo());+ descriptor.setLocation(spec.location());+ Partition partition = new Partition();+ partition.setValues(values);+ partition.setStorageDescriptor(descriptor);+ return partition;+ }++ private static boolean isOpenCsv(String serde) {+ return serde != null && serde.toLowerCase(Locale.ROOT).contains("opencsvserde");+ }++ private static List<Column> columns(List<ExternalStatement.ColumnDefinition> definitions) {+ List<Column> result = new ArrayList<>();+ for (ExternalStatement.ColumnDefinition definition : definitions) {+ Column column = new Column();+ column.setName(definition.name());+ column.setType(definition.type().toLowerCase(Locale.ROOT));+ result.add(column);+ }+ return result;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapper.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapper.javanew file mode 100644index 0000000000..92a2305162--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapper.java@@ -0,0 +1,111 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import java.util.Locale;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++/** Maps Glue column types to PostgreSQL types and DuckDB text projections. */+public final class GlueTypeMapper {+ private static final Pattern SIZED_STRING = Pattern.compile("(?:varchar|char)\\((\\d+)\\)");+ private static final Pattern DECIMAL = Pattern.compile("(?:decimal|numeric)(?:\\((\\d+)(?:,(\\d+))?\\))?");+ private static final String NULL_MARKER = "'\\N'";++ private GlueTypeMapper() {+ }++ /** Rewrites Redshift type aliases to the Hive type names that Glue stores; other types pass through lowercased. */+ public static String canonicalGlueType(String type) {+ String normalized = normalize(type);+ if (isTypeOrSized(normalized, "charactervarying")) {+ String length = normalized.substring("charactervarying".length());+ return "varchar" + (length.isEmpty() ? "(256)" : length);+ }+ if (isTypeOrSized(normalized, "character")) {+ String length = normalized.substring("character".length());+ return "char" + (length.isEmpty() ? "(1)" : length);+ }+ if (isTypeOrSized(normalized, "numeric")) {+ return "decimal" + normalized.substring("numeric".length());+ }+ if (isBinary(normalized)) {+ return "binary";+ }+ return switch (normalized) {+ case "varchar" -> "varchar(256)";+ case "char" -> "char(1)";+ case "doubleprecision", "float8" -> "double";+ case "real", "float4" -> "float";+ case "int2" -> "smallint";+ case "int4" -> "int";+ case "int8" -> "bigint";+ case "bool" -> "boolean";+ default -> normalized;+ };+ }++ public static String toPostgres(String glueType) {+ String type = canonicalGlueType(glueType);+ Matcher sized = SIZED_STRING.matcher(type);+ if (sized.matches()) {+ return "varchar(" + sized.group(1) + ")";+ }+ Matcher decimal = DECIMAL.matcher(type);+ if (decimal.matches()) {+ if (decimal.group(1) == null) {+ return "numeric";+ }+ return "numeric(" + decimal.group(1) + "," + (decimal.group(2) == null ? "0" : decimal.group(2)) + ")";+ }+ if (isNested(type)) {+ return "jsonb";+ }+ return switch (type) {+ case "string", "varchar", "char" -> "text";+ case "tinyint", "smallint" -> "smallint";+ case "int", "integer" -> "integer";+ case "bigint" -> "bigint";+ case "float" -> "real";+ case "double" -> "double precision";+ case "boolean" -> "boolean";+ case "date" -> "date";+ case "timestamp" -> "timestamp";+ case "binary" -> "bytea";+ default -> throw new SpectrumSqlException("0A000", "Unsupported external column type: " + glueType);+ };+ }++ public static boolean isNested(String glueType) {+ String type = normalize(glueType);+ return type.startsWith("array<") || type.startsWith("map<") || type.startsWith("struct<");+ }++ public static String duckProjection(String columnName, String glueType) {+ String column = quote(columnName);+ String type = canonicalGlueType(glueType);+ String expression = isNested(type) ? "to_json(" + column + ")"+ : "binary".equals(type) ? "'\\x' || hex(" + column + ")" : column;+ return "COALESCE(CAST(" + expression + " AS VARCHAR), " + NULL_MARKER + ") AS " + column;+ }++ private static boolean isTypeOrSized(String type, String base) {+ return type.equals(base) || type.startsWith(base + "(");+ }++ private static boolean isBinary(String type) {+ return isTypeOrSized(type, "varbyte")+ || isTypeOrSized(type, "varbinary")+ || isTypeOrSized(type, "binaryvarying")+ || isTypeOrSized(type, "binary");+ }++ private static String normalize(String glueType) {+ if (glueType == null) {+ throw new SpectrumSqlException("0A000", "External column type is missing");+ }+ return glueType.trim().toLowerCase(Locale.ROOT).replace(" ", "");+ }++ private static String quote(String identifier) {+ return "\"" + identifier.replace("\"", "\"\"") + "\"";+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSession.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSession.javanew file mode 100644index 0000000000..b0073ff3df--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSession.java@@ -0,0 +1,173 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import io.github.hectorvent.floci.services.redshift.proxy.PostgresWireDecoder;++import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.Socket;+import java.nio.charset.StandardCharsets;++public final class PostgresBackendSession implements BackendSql {+ private static final String SQLSTATE_IO = "58030";+ private final Socket backend;++ public PostgresBackendSession(Socket backend) {+ this.backend = backend;+ }++ @Override+ public void execute(String sql) {+ try {+ OutputStream output = backend.getOutputStream();+ output.write(PostgresWireDecoder.encodeQuery(sql));+ output.flush();+ awaitReady(new PostgresWireDecoder(backend.getInputStream()));+ } catch (IOException exception) {+ throw new SpectrumReadException(SQLSTATE_IO, "Unable to run a statement on the Redshift backend", exception);+ }+ }++ @Override+ public long copyIn(String copySql, InputStream data) {+ try {+ OutputStream output = backend.getOutputStream();+ output.write(PostgresWireDecoder.encodeQuery(copySql));+ output.flush();+ PostgresWireDecoder decoder = new PostgresWireDecoder(backend.getInputStream());+ awaitCopyIn(decoder);+ byte[] chunk = new byte[8192];+ int read = readChunk(output, decoder, data, chunk);+ while (read > 0) {+ output.write('d');+ writeInt32(output, read + 4);+ output.write(chunk, 0, read);+ read = readChunk(output, decoder, data, chunk);+ }+ output.write('c');+ writeInt32(output, 4);+ output.flush();+ return rowCount(awaitReady(decoder));+ } catch (IOException exception) {+ throw new SpectrumReadException(SQLSTATE_IO, "Unable to load external table data", exception);+ }+ }++ /** A failure reading the source aborts the COPY on the backend; a failure writing to the backend socket does not. */+ private static int readChunk(OutputStream output, PostgresWireDecoder decoder, InputStream data, byte[] chunk)+ throws IOException {+ try {+ int read = data.read(chunk);+ if (read != 0) {+ return read;+ }+ int nextByte = data.read();+ if (nextByte < 0) {+ return -1;+ }+ chunk[0] = (byte) nextByte;+ return 1;+ } catch (IOException exception) {+ writeCopyFail(output, exception.getMessage());+ try {+ awaitReady(decoder);+ } catch (IOException | SpectrumReadException backendFailure) {+ exception.addSuppressed(backendFailure);+ }+ throw exception;+ }+ }++ private static void awaitCopyIn(PostgresWireDecoder decoder) throws IOException {+ SpectrumReadException failure = null;+ PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();+ while (message != null) {+ if (message.type() == 'G') {+ if (failure != null) {+ throw failure;+ }+ return;+ }+ if (message.type() == 'E') {+ failure = toException(message.body());+ } else if (message.type() == 'Z') {+ if (failure != null) {+ throw failure;+ }+ throw new IOException("PostgreSQL did not enter CopyIn mode");+ }+ message = decoder.nextMessage();+ }+ throw new IOException("PostgreSQL closed before CopyInResponse");+ }++ private static String awaitReady(PostgresWireDecoder decoder) throws IOException {+ SpectrumReadException failure = null;+ String tag = "";+ PostgresWireDecoder.FrontendMessage message = decoder.nextMessage();+ while (message != null) {+ if (message.type() == 'C') {+ tag = new String(message.body(), StandardCharsets.UTF_8).trim();+ } else if (message.type() == 'E') {+ failure = toException(message.body());+ } else if (message.type() == 'Z') {+ if (failure != null) {+ throw failure;+ }+ return tag;+ }+ message = decoder.nextMessage();+ }+ throw new IOException("PostgreSQL closed before ReadyForQuery");+ }++ private static long rowCount(String tag) {+ int space = tag.lastIndexOf(' ');+ if (space < 0) {+ return 0;+ }+ try {+ return Long.parseLong(tag.substring(space + 1).trim());+ } catch (NumberFormatException exception) {+ throw new SpectrumReadException(SQLSTATE_IO, "Unexpected COPY command tag: " + tag, exception);+ }+ }++ private static SpectrumReadException toException(byte[] body) {+ String state = SQLSTATE_IO;+ String message = "PostgreSQL rejected the statement";+ int offset = 0;+ while (body != null && offset < body.length && body[offset] != 0) {+ char code = (char) body[offset++];+ int end = offset;+ while (end < body.length && body[end] != 0) {+ end++;+ }+ String value = new String(body, offset, end - offset, StandardCharsets.UTF_8);+ if (code == 'C') {+ state = value;+ } else if (code == 'M') {+ message = value;+ }+ offset = end + 1;+ }+ return new SpectrumReadException(state, message);+ }++ private static void writeInt32(OutputStream output, int value) throws IOException {+ output.write((value >>> 24) & 0xff);+ output.write((value >>> 16) & 0xff);+ output.write((value >>> 8) & 0xff);+ output.write(value & 0xff);+ }++ private static void writeCopyFail(OutputStream output, String message) throws IOException {+ byte[] detail = (message == null ? "Unable to stream COPY input" : message)+ .getBytes(StandardCharsets.UTF_8);+ output.write('f');+ writeInt32(output, detail.length + 5);+ output.write(detail);+ output.write(0);+ output.flush();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumColumn.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumColumn.javaindex 5427eae299..3274e72f84 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumColumn.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumColumn.java@@ -50,7 +50,7 @@ public record SpectrumColumn(String name, Type type) {public String postgresType() {return switch (this) {- case VARCHAR -> "VARCHAR";+ case VARCHAR -> "VARCHAR(256)";case CHAR -> "CHAR";case INTEGER -> "INTEGER";case BIGINT -> "BIGINT";diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.javaindex 5d38129689..a6a0384882 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializer.java@@ -21,6 +21,7 @@ public final class SpectrumMaterializer {private static final SecureRandom RANDOM = new SecureRandom();private static final String SQLSTATE_DATA = "22000";+ private static final int DEFAULT_VARCHAR_MAX_BYTES = 256;public Materialization materialize(Socket backend, SpectrumExternalTable table,SpectrumExternalSchema schema, SpectrumS3Reader reader) {@@ -44,7 +45,7 @@ public final class SpectrumMaterializer {try (Stream<SpectrumRow> rows = reader.read(schema, table)) {Iterator<SpectrumRow> iterator = rows.iterator();while (iterator.hasNext()) {- writeCopyData(output, encodeRow(iterator.next()));+ writeCopyData(output, encodeRow(iterator.next(), table.columns()));}}writeCopyDone(output);@@ -96,13 +97,18 @@ public final class SpectrumMaterializer {+ ") FROM STDIN";}- private static byte[] encodeRow(SpectrumRow row) {+ private static byte[] encodeRow(SpectrumRow row, List<SpectrumColumn> columns) {StringBuilder line = new StringBuilder();for (int i = 0; i < row.values().size(); i++) {if (i > 0) {line.append('\t');}String value = row.values().get(i);+ if (value != null+ && columns.get(i).type() == SpectrumColumn.Type.VARCHAR+ && value.getBytes(StandardCharsets.UTF_8).length > DEFAULT_VARCHAR_MAX_BYTES) {+ value = null;+ }if (value == null) {line.append("\\N");} else {diff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumSession.java b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumSession.javanew file mode 100644index 0000000000..c9d9c83577--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumSession.java@@ -0,0 +1,10 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import java.util.List;++public record SpectrumSession(String accountId, String clusterKey, String databaseName,+ List<String> iamRoleArns, boolean inTransaction) {+ public SpectrumSession {+ iamRoleArns = iamRoleArns == null ? List.of() : List.copyOf(iamRoleArns);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistryTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistryTest.javanew file mode 100644index 0000000000..2436fb113a--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalCatalogRegistryTest.java@@ -0,0 +1,79 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.contains;+import static org.hamcrest.Matchers.empty;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++class ExternalCatalogRegistryTest {+ private static final String ACCOUNT = "000000000000";+ private ExternalCatalogRegistry registry;++ @BeforeEach+ @SuppressWarnings("unchecked")+ void setUp() {+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.create(eq("redshift"), any(String.class), any(TypeReference.class)))+ .thenReturn(new AccountAwareStorageBackend<ExternalSchemaBinding>(new InMemoryStorage<>(), null, ACCOUNT));+ registry = new ExternalCatalogRegistry(factory);+ }++ private static ExternalSchemaBinding binding(String cluster, String database, String schema) {+ return new ExternalSchemaBinding(ACCOUNT, cluster, database, schema, "lake", "arn:aws:iam::000000000000:role/R");+ }++ @Test+ void bindsFindsListsAndUnbinds() {+ registry.bind(binding("c1", "dev", "b"));+ registry.bind(binding("c1", "dev", "a"));++ assertThat(registry.find(ACCOUNT, "c1", "dev", "a").isPresent(), equalTo(true));+ assertThat(registry.list(ACCOUNT, "c1", "dev").stream().map(ExternalSchemaBinding::schemaName).toList(),+ contains("a", "b"));+ registry.unbind(ACCOUNT, "c1", "dev", "a");+ assertThat(registry.find(ACCOUNT, "c1", "dev", "a").isPresent(), equalTo(false));+ }++ @Test+ void rejectsDuplicateSchema() {+ registry.bind(binding("c1", "dev", "a"));++ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,+ () -> registry.bind(binding("c1", "dev", "a")));+ assertThat(exception.sqlState(), equalTo("42P06"));+ }++ @Test+ void identifiersContainingTheSeparatorDoNotCollideOrLeakAcrossScopes() {+ registry.bind(binding("c1", "dev|x", "s"));+ registry.bind(binding("c1", "dev", "x|s"));++ assertThat(registry.list(ACCOUNT, "c1", "dev").stream().map(ExternalSchemaBinding::schemaName).toList(),+ contains("x|s"));+ assertThat(registry.list(ACCOUNT, "c1", "dev|x").stream().map(ExternalSchemaBinding::schemaName).toList(),+ contains("s"));+ }++ @Test+ void removeClusterDropsOnlyThatCluster() {+ registry.bind(binding("c1", "dev", "a"));+ registry.bind(binding("c10", "dev", "a"));++ registry.removeCluster(ACCOUNT, "c1");++ assertThat(registry.list(ACCOUNT, "c1", "dev"), empty());+ assertThat(registry.list(ACCOUNT, "c10", "dev").size(), equalTo(1));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriterTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriterTest.javanew file mode 100644index 0000000000..27c5299e1d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalMetadataWriterTest.java@@ -0,0 +1,182 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.services.glue.GlueService;+import io.github.hectorvent.floci.services.glue.model.Column;+import io.github.hectorvent.floci.services.glue.model.Partition;+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;+import io.github.hectorvent.floci.services.glue.model.Table;+import org.junit.jupiter.api.Test;++import java.io.InputStream;+import java.util.List;+import java.util.Map;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.Mockito.mock;++class ExternalMetadataWriterTest {+ private static final ExternalSchemaBinding BINDING = new ExternalSchemaBinding("000000000000",+ "000000000000:c", "dev", "analytics", "lake", "arn:aws:iam::000000000000:role/R");+ private final ExternalMetadataWriter writer = new ExternalMetadataWriter(mock(GlueService.class), new ObjectMapper());++ @Test+ void refreshSqlCallsPrivilegedWriterWithGlueMetadataPayload() {+ Column id = new Column();+ id.setName("id");+ id.setType("int");+ Column dt = new Column();+ dt.setName("dt");+ dt.setType("string");+ StorageDescriptor descriptor = new StorageDescriptor();+ descriptor.setLocation("s3://bucket/o'brien/");+ descriptor.setColumns(List.of(id));+ Table table = new Table();+ table.setName("events");+ table.setStorageDescriptor(descriptor);+ table.setPartitionKeys(List.of(dt));+ table.setParameters(Map.of("EXTERNAL", "TRUE"));+ Partition partition = new Partition();+ partition.setValues(List.of("2024-01-01"));++ String sql = writer.refreshSql(BINDING, List.of(table), Map.of("events", List.of(partition)));++ assertThat(sql, containsString("SELECT floci_internal.refresh_external_catalog('analytics'"));+ assertThat(sql, containsString("IAM_ROLE"));+ assertThat(sql, containsString("\"location\":\"s3://bucket/o''brien/\""));+ assertThat(sql, containsString("\"columnname\":\"id\",\"external_type\":\"int\""));+ assertThat(sql, containsString("\"columnname\":\"dt\",\"external_type\":\"string\""));+ assertThat(sql, containsString("2024-01-01"));+ assertThat(sql.contains("INSERT INTO floci_internal."), equalTo(false));+ }++ @Test+ void refreshSqlUsesPartitionStorageSettings() throws Exception {+ StorageDescriptor tableDescriptor = new StorageDescriptor();+ tableDescriptor.setSerdeInfo(serdeInfo("table-serde", Map.of("source", "table")));+ tableDescriptor.setCompressed(false);+ Table table = new Table();+ table.setName("events");+ table.setStorageDescriptor(tableDescriptor);++ StorageDescriptor partitionDescriptor = new StorageDescriptor();+ partitionDescriptor.setSerdeInfo(serdeInfo("partition-serde", Map.of("source", "partition")));+ partitionDescriptor.setCompressed(true);+ Partition partition = new Partition();+ partition.setValues(List.of("2024-01-01"));+ partition.setStorageDescriptor(partitionDescriptor);++ String sql = writer.refreshSql(BINDING, List.of(table), Map.of("events", List.of(partition)));+ JsonNode partitionRow = refreshPayload(sql).path("partitions").get(0);++ assertEquals("partition-serde", partitionRow.path("serialization_lib").asText());+ assertEquals("{\"source\":\"partition\"}", partitionRow.path("serde_parameters").asText());+ assertEquals(1, partitionRow.path("compressed").asInt());+ }++ @Test+ void refreshSqlFallsBackToTableSettingsOmittedFromPartitionDescriptor() throws Exception {+ StorageDescriptor tableDescriptor = new StorageDescriptor();+ tableDescriptor.setSerdeInfo(serdeInfo("table-serde", Map.of("source", "table")));+ tableDescriptor.setCompressed(true);+ Table table = new Table();+ table.setName("events");+ table.setStorageDescriptor(tableDescriptor);++ StorageDescriptor partitionDescriptor = new StorageDescriptor();+ partitionDescriptor.setLocation("s3://bucket/events/date=2024-01-01/");+ Partition partition = new Partition();+ partition.setValues(List.of("2024-01-01"));+ partition.setStorageDescriptor(partitionDescriptor);++ String sql = writer.refreshSql(BINDING, List.of(table), Map.of("events", List.of(partition)));+ JsonNode partitionRow = refreshPayload(sql).path("partitions").get(0);++ assertEquals("table-serde", partitionRow.path("serialization_lib").asText());+ assertEquals("{\"source\":\"table\"}", partitionRow.path("serde_parameters").asText());+ assertEquals(1, partitionRow.path("compressed").asInt());+ }++ @Test+ void refreshSqlPreservesExplicitlyEmptyPartitionSerdeParameters() throws Exception {+ StorageDescriptor tableDescriptor = new StorageDescriptor();+ tableDescriptor.setSerdeInfo(serdeInfo("table-serde", Map.of("field.delim", "|")));+ Table table = new Table();+ table.setName("events");+ table.setStorageDescriptor(tableDescriptor);++ StorageDescriptor partitionDescriptor = new StorageDescriptor();+ partitionDescriptor.setSerdeInfo(serdeInfo("partition-serde", Map.of()));+ Partition partition = new Partition();+ partition.setValues(List.of("2024-01-01"));+ partition.setStorageDescriptor(partitionDescriptor);++ String sql = writer.refreshSql(BINDING, List.of(table), Map.of("events", List.of(partition)));+ JsonNode partitionRow = refreshPayload(sql).path("partitions").get(0);++ assertEquals("{}", partitionRow.path("serde_parameters").asText());+ }++ @Test+ void purgeCallsPrivilegedWriterForOneSchema() {+ String sql = writer.purgeSql("analytics");+ assertThat(sql, containsString("SELECT floci_internal.purge_external_schema('analytics')"));+ assertThat(sql.contains("DELETE FROM floci_internal."), equalTo(false));+ }++ @Test+ void purgeToleratesMissingBootstrapTables() {+ BackendSql missingTable = new BackendSql() {+ @Override+ public void execute(String sql) {+ throw new SpectrumReadException("42P01", "metadata table missing");+ }++ @Override+ public long copyIn(String copySql, InputStream data) {+ return 0;+ }+ };+ assertDoesNotThrow(() -> writer.purge(missingTable, "analytics"));+ }++ @Test+ void refreshPropagatesMissingBootstrapTables() {+ BackendSql missingTable = new BackendSql() {+ @Override+ public void execute(String sql) {+ throw new SpectrumReadException("42P01", "metadata table missing");+ }++ @Override+ public long copyIn(String copySql, InputStream data) {+ return 0;+ }+ };++ SpectrumReadException exception = assertThrows(SpectrumReadException.class,+ () -> writer.refresh(missingTable, BINDING.accountId(), BINDING));++ assertEquals("42P01", exception.sqlState());+ }++ private static StorageDescriptor.SerDeInfo serdeInfo(String library, Map<String, String> parameters) {+ StorageDescriptor.SerDeInfo serdeInfo = new StorageDescriptor.SerDeInfo();+ serdeInfo.setSerializationLibrary(library);+ serdeInfo.setParameters(parameters);+ return serdeInfo;+ }++ private JsonNode refreshPayload(String sql) throws Exception {+ String prefix = "SELECT floci_internal.refresh_external_catalog('analytics', '";+ String payloadEnd = "'::jsonb)";+ String escapedJson = sql.substring(prefix.length(), sql.lastIndexOf(payloadEnd));+ return new ObjectMapper().readTree(escapedJson.replace("''", "'"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScannerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScannerTest.javanew file mode 100644index 0000000000..df2b92a078--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalReferenceScannerTest.java@@ -0,0 +1,186 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import io.github.hectorvent.floci.services.redshift.spectrum.ExternalReferenceScanner.Reference;+import org.junit.jupiter.api.Test;++import java.util.Optional;+import java.util.Set;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.contains;+import static org.hamcrest.Matchers.empty;+import static org.hamcrest.Matchers.equalTo;++class ExternalReferenceScannerTest {+ private static final Set<String> SCHEMAS = Set.of("analytics", "Mixed");++ @Test+ void findsQualifiedReferencesAndIgnoresOtherSchemas() {+ assertThat(ExternalReferenceScanner.scan("SELECT * FROM analytics.events e JOIN public.users u ON true", SCHEMAS),+ contains(new Reference("analytics", "events")));+ }++ @Test+ void keepsQuotedCaseAndSupportsDatabaseQualifiedNames() {+ assertThat(ExternalReferenceScanner.scan("SELECT * FROM dev.analytics.events, \"Mixed\".\"Order\"", SCHEMAS),+ contains(new Reference("analytics", "events"), new Reference("Mixed", "Order")));+ }++ @Test+ void findsQualifiedReferencesWithWhitespaceOrCommentsAroundDots() {+ assertThat(ExternalReferenceScanner.scan(+ "SELECT * FROM analytics /* schema */ .\n events", SCHEMAS),+ contains(new Reference("analytics", "events")));+ }++ @Test+ void ignoresStringsAndComments() {+ assertThat(ExternalReferenceScanner.scan("SELECT 'analytics.events' -- analytics.other\n /* analytics.x */", SCHEMAS), empty());+ }++ @Test+ void detectsOnlyWritesTargetingExternalTables() {+ assertThat(ExternalReferenceScanner.writeTarget("INSERT INTO analytics.events VALUES (1)", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ assertThat(ExternalReferenceScanner.writeTarget("INSERT INTO native_copy SELECT * FROM analytics.events", SCHEMAS), equalTo(Optional.empty()));+ assertThat(ExternalReferenceScanner.writeTarget("WITH values_to_write AS (SELECT 1) UPDATE analytics.events SET id = 2", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ }++ @Test+ void doesNotTreatColumnQualifiersOrTableAliasesAsTableReferences() {+ assertThat(ExternalReferenceScanner.scan("SELECT analytics.id FROM public.owners AS analytics", SCHEMAS), empty());+ assertThat(ExternalReferenceScanner.scan("SELECT * FROM public.owners analytics WHERE analytics.id = 1", SCHEMAS), empty());+ assertThat(ExternalReferenceScanner.scan(+ "SELECT * FROM public.owners o JOIN public.teams t ON t.id = analytics.id", SCHEMAS), empty());+ }++ @Test+ void findsCommaJoinedAndSubqueryReferencesAndRestoresContextAfterTheSubquery() {+ assertThat(ExternalReferenceScanner.scan("SELECT * FROM public.a, analytics.events", SCHEMAS),+ contains(new Reference("analytics", "events")));+ assertThat(ExternalReferenceScanner.scan(+ "SELECT * FROM public.t WHERE id IN (SELECT id FROM analytics.events)", SCHEMAS),+ contains(new Reference("analytics", "events")));+ assertThat(ExternalReferenceScanner.scan(+ "SELECT * FROM public.t WHERE id IN (SELECT id FROM public.u) AND analytics.id = 1", SCHEMAS), empty());+ }++ @Test+ void checksEveryStatementOfABatchForWrites() {+ assertThat(ExternalReferenceScanner.writeTarget(+ "UPDATE public.owners SET x = 1; DELETE FROM analytics.events", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ }++ @Test+ void detectsWritesInsideDataModifyingCtesTruncateAndMerge() {+ assertThat(ExternalReferenceScanner.writeTarget(+ "WITH gone AS (DELETE FROM analytics.events RETURNING id) SELECT * FROM gone", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ assertThat(ExternalReferenceScanner.writeTarget("TRUNCATE TABLE analytics.events", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ assertThat(ExternalReferenceScanner.writeTarget(+ "TRUNCATE TABLE public.local_table, analytics.events", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ assertThat(ExternalReferenceScanner.writeTarget("MERGE INTO ONLY analytics.events USING public.t ON true", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ assertThat(ExternalReferenceScanner.writeTarget(+ "MERGE INTO analytics.events USING public.t ON true WHEN MATCHED THEN DELETE", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ }++ @Test+ void detectsDdlThatWouldChangeAnExternalTableBehindFlocisBack() {+ Optional<Reference> events = Optional.of(new Reference("analytics", "events"));+ assertThat(ExternalReferenceScanner.writeTarget("DROP TABLE public.a, analytics.events", SCHEMAS), equalTo(events));+ assertThat(ExternalReferenceScanner.writeTarget("SELECT 1; DROP TABLE IF EXISTS analytics.events", SCHEMAS), equalTo(events));+ assertThat(ExternalReferenceScanner.writeTarget("ALTER TABLE analytics.events RENAME TO x", SCHEMAS), equalTo(events));+ assertThat(ExternalReferenceScanner.writeTarget("ALTER TABLE IF EXISTS ONLY analytics.events ADD COLUMN c int", SCHEMAS), equalTo(events));+ assertThat(ExternalReferenceScanner.writeTarget("COPY analytics.events FROM 's3://b/k'", SCHEMAS), equalTo(events));+ assertThat(ExternalReferenceScanner.writeTarget("CREATE TABLE analytics.events (id int)", SCHEMAS), equalTo(events));+ assertThat(ExternalReferenceScanner.writeTarget("CREATE TEMP TABLE IF NOT EXISTS analytics.events (id int)", SCHEMAS), equalTo(events));+ }++ @Test+ void detectsDroppingAnExternalSchemaFromAList() {+ assertThat(ExternalReferenceScanner.writeTarget("DROP SCHEMA local_schema, analytics CASCADE", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", ""))));+ }++ @Test+ void ddlOnNativeObjectsIsNotAnExternalWrite() {+ assertThat(ExternalReferenceScanner.writeTarget("DROP TABLE public.a, public.b", SCHEMAS), equalTo(Optional.empty()));+ assertThat(ExternalReferenceScanner.writeTarget("DROP SCHEMA local_schema", SCHEMAS), equalTo(Optional.empty()));+ assertThat(ExternalReferenceScanner.writeTarget("ALTER TABLE public.t ADD COLUMN c int", SCHEMAS), equalTo(Optional.empty()));+ assertThat(ExternalReferenceScanner.writeTarget("CREATE TABLE public.t AS SELECT * FROM analytics.events", SCHEMAS), equalTo(Optional.empty()));+ assertThat(ExternalReferenceScanner.writeTarget("COPY public.t FROM 's3://b/k'", SCHEMAS), equalTo(Optional.empty()));+ }++ @Test+ void detectsNativeTablesRenamedOrMovedIntoAnExternalSchema() {+ assertThat(ExternalReferenceScanner.writeTarget("ALTER TABLE public.t RENAME TO analytics.events", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ assertThat(ExternalReferenceScanner.writeTarget("ALTER TABLE public.t SET SCHEMA analytics", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", ""))));+ }++ @Test+ void writesToNativeTablesAreNotExternalWrites() {+ assertThat(ExternalReferenceScanner.writeTarget("UPDATE public.owners SET x = 1", SCHEMAS), equalTo(Optional.empty()));+ assertThat(ExternalReferenceScanner.writeTarget(+ "INSERT INTO public.t VALUES (1) ON CONFLICT DO UPDATE SET x = 1", SCHEMAS), equalTo(Optional.empty()));+ }++ @Test+ void writeTargetSeesPastDollarQuotedBodiesAndEscapeStrings() {+ assertThat(ExternalReferenceScanner.writeTarget(+ "SELECT $$ it's $$; INSERT INTO analytics.events VALUES (1)", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ assertThat(ExternalReferenceScanner.writeTarget(+ "SELECT $tag$ it's $$ still inside $tag$; DELETE FROM analytics.events", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ assertThat(ExternalReferenceScanner.writeTarget(+ "SELECT E'it\\'s'; INSERT INTO analytics.events VALUES (1)", SCHEMAS),+ equalTo(Optional.of(new Reference("analytics", "events"))));+ }++ @Test+ void ignoresReferencesInsideDollarQuotesAndNestedComments() {+ assertThat(ExternalReferenceScanner.scan("SELECT $$ FROM analytics.events $$", SCHEMAS), empty());+ assertThat(ExternalReferenceScanner.scan("SELECT 1 /* a /* FROM analytics.events */ FROM analytics.other */", SCHEMAS), empty());+ }++ @Test+ void positionalParametersAreNotDollarQuotes() {+ assertThat(ExternalReferenceScanner.scan("SELECT $1 FROM analytics.events WHERE id = $2", SCHEMAS),+ contains(new Reference("analytics", "events")));+ }++ @Test+ void detectsSelectIntoViewAndIndexTargets() {+ Optional<Reference> expected = Optional.of(new Reference("analytics", "events"));+ assertThat(ExternalReferenceScanner.writeTarget("SELECT * INTO analytics.events FROM public.t", SCHEMAS), equalTo(expected));+ assertThat(ExternalReferenceScanner.writeTarget("CREATE OR REPLACE VIEW analytics.events AS SELECT 1", SCHEMAS), equalTo(expected));+ assertThat(ExternalReferenceScanner.writeTarget("CREATE MATERIALIZED VIEW analytics.events AS SELECT 1", SCHEMAS), equalTo(expected));+ assertThat(ExternalReferenceScanner.writeTarget("CREATE UNIQUE INDEX i ON analytics.events (id)", SCHEMAS), equalTo(expected));+ assertThat(ExternalReferenceScanner.writeTarget("CREATE INDEX i ON public.t (id)", SCHEMAS), equalTo(Optional.empty()));+ assertThat(ExternalReferenceScanner.writeTarget("SELECT * INTO public.copy FROM analytics.events", SCHEMAS), equalTo(Optional.empty()));+ }++ @Test+ void fromInsideExpressionFunctionsIsNotATableList() {+ assertThat(ExternalReferenceScanner.scan("SELECT EXTRACT(year FROM analytics.ts) FROM public.t", SCHEMAS), empty());+ assertThat(ExternalReferenceScanner.scan("SELECT SUBSTRING(a FROM analytics.b), TRIM(BOTH 'x' FROM analytics.c) FROM public.t", SCHEMAS), empty());+ assertThat(ExternalReferenceScanner.scan("SELECT * FROM public.t WHERE a IS DISTINCT FROM analytics.b", SCHEMAS), empty());+ }++ @Test+ void tableListInsideExpressionFunctionStillCounts() {+ assertThat(ExternalReferenceScanner.scan(+ "SELECT EXTRACT(year FROM (SELECT max(ts) FROM analytics.events)) FROM public.t", SCHEMAS),+ contains(new Reference("analytics", "events")));+ assertThat(ExternalReferenceScanner.scan("SELECT EXTRACT(year FROM ts) FROM analytics.events", SCHEMAS),+ contains(new Reference("analytics", "events")));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParserTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParserTest.javanew file mode 100644index 0000000000..968a91b265--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/ExternalStatementParserTest.java@@ -0,0 +1,242 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import org.junit.jupiter.api.Test;++import java.util.List;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertThrows;++class ExternalStatementParserTest {+ private final ExternalStatementParser parser = new ExternalStatementParser();++ @Test+ void parsesCreateExternalSchema() {+ ExternalStatement.CreateSchema schema = (ExternalStatement.CreateSchema) parser.parse(+ "CREATE EXTERNAL SCHEMA Analytics FROM DATA CATALOG DATABASE 'lake' REGION 'us-east-1' IAM_ROLE 'arn:aws:iam::000000000000:role/R' CREATE EXTERNAL DATABASE IF NOT EXISTS").orElseThrow();+ assertThat(schema.schemaName(), equalTo("analytics"));+ assertThat(schema.glueDatabase(), equalTo("lake"));+ assertThat(schema.createDatabaseIfNotExists(), equalTo(true));+ }++ @Test+ void parsesCreateTableWithNestedTypesAndFormat() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (id BIGINT, tags array<string>) STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();+ assertThat(table.format(), equalTo(ExternalStatement.TableFormat.PARQUET));+ assertThat(table.columns().get(1).type(), equalTo("array<string>"));+ }++ @Test+ void parsesPartitionColumnsWhoseTypesContainParentheses() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (id BIGINT) PARTITIONED BY (price decimal(10,2), day string) "+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();+ assertThat(table.partitionColumns().size(), equalTo(2));+ assertThat(table.partitionColumns().get(0).type(), equalTo("decimal(10,2)"));+ assertThat(table.partitionColumns().get(1).name(), equalTo("day"));+ }++ @Test+ void parsesSerdePropertiesOfAnOpenCsvTable() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (id INT, note VARCHAR(20)) "+ + "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' "+ + "WITH SERDEPROPERTIES ('separatorChar'='|', 'quoteChar'='\"') "+ + "STORED AS TEXTFILE LOCATION 's3://bucket/events/'").orElseThrow();+ assertThat(table.serde(), equalTo("org.apache.hadoop.hive.serde2.OpenCSVSerde"));+ assertThat(table.serdeProperties().get("separatorChar"), equalTo("|"));+ assertThat(table.serdeProperties().get("quoteChar"), equalTo("\""));+ }++ @Test+ void parsesSerdePropertyValuesThatContainClosingParentheses() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (id INT) "+ + "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' "+ + "WITH SERDEPROPERTIES ('separatorChar'=')', 'quoteChar'='(') "+ + "STORED AS TEXTFILE LOCATION 's3://bucket/events/'").orElseThrow();++ assertThat(table.serdeProperties().get("separatorChar"), equalTo(")"));+ assertThat(table.serdeProperties().get("quoteChar"), equalTo("("));+ }++ @Test+ void parsesColumnNamesContainingCommasWhenQuoted() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (\"event,id\" VARCHAR, amount DECIMAL(10,2)) "+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();++ assertThat(table.columns().stream().map(ExternalStatement.ColumnDefinition::name).toList(),+ equalTo(List.of("event,id", "amount")));+ }++ @Test+ void parsesQuotedColumnNamesContainingSpaces() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (\"event name\" VARCHAR(20), amount DECIMAL(10,2)) "+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();++ assertThat(table.columns().stream().map(ExternalStatement.ColumnDefinition::name).toList(),+ equalTo(List.of("event name", "amount")));+ assertThat(table.columns().get(0).type(), equalTo("varchar(20)"));+ }++ @Test+ void parsesCreateTableWithCharacterAndVarbyteColumns() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (code CHARACTER(10), payload VARBYTE, memo CHARACTER) "+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();+ assertThat(table.columns().get(0).name(), equalTo("code"));+ assertThat(table.columns().get(0).type(), equalTo("char(10)"));+ assertThat(table.columns().get(1).name(), equalTo("payload"));+ assertThat(table.columns().get(1).type(), equalTo("binary"));+ assertThat(table.columns().get(2).name(), equalTo("memo"));+ assertThat(table.columns().get(2).type(), equalTo("char(1)"));+ }++ @Test+ void defaultsUnsizedCharacterVaryingColumnsToRedshiftLength() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (memo CHARACTER VARYING) "+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();++ assertThat(table.columns().get(0).type(), equalTo("varchar(256)"));+ }++ @Test+ void createDatabaseOptionSurvivesLineBreaksAndExtraWhitespace() {+ ExternalStatement.CreateSchema schema = (ExternalStatement.CreateSchema) parser.parse(+ "CREATE EXTERNAL SCHEMA a FROM DATA CATALOG DATABASE 'lake' IAM_ROLE 'arn:aws:iam::000000000000:role/R'\n"+ + "CREATE EXTERNAL\n DATABASE IF NOT EXISTS").orElseThrow();+ assertThat(schema.createDatabaseIfNotExists(), equalTo(true));+ }++ @Test+ void otherCreateExternalStatementsAreReportedAsUnsupported() {+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class,+ () -> parser.parse("CREATE EXTERNAL FUNCTION f(int) RETURNS int LAMBDA 'x' IAM_ROLE 'r'"));+ assertThat(error.sqlState(), equalTo("0A000"));+ assertThat(error.getMessage().contains("unsupported"), equalTo(true));+ }++ @Test+ void parsesMultiplePartitionClausesAndPreservesQuotedNames() {+ ExternalStatement.AddPartitions add = (ExternalStatement.AddPartitions) parser.parse(+ "ALTER TABLE \"Mixed\".\"Events\" ADD IF NOT EXISTS "+ + "PARTITION (dt='2024-01-01', region='eu') LOCATION 's3://b/one/' "+ + "PARTITION (dt='2024-01-02', region='eu') LOCATION 's3://b/two/'").orElseThrow();+ assertThat(add.schemaName(), equalTo("Mixed"));+ assertThat(add.tableName(), equalTo("Events"));+ assertThat(add.ifNotExists(), equalTo(true));+ assertThat(add.partitions().size(), equalTo(2));+ assertThat(add.partitions().get(1).values().get("dt"), equalTo("2024-01-02"));+ }++ @Test+ void rejectsUnsupportedRoleDefaultAndNonS3Location() {+ SpectrumSqlException role = assertThrows(SpectrumSqlException.class,+ () -> parser.parse("CREATE EXTERNAL SCHEMA a FROM DATA CATALOG DATABASE 'd' IAM_ROLE default"));+ assertThat(role.sqlState(), equalTo("0A000"));+ SpectrumSqlException location = assertThrows(SpectrumSqlException.class,+ () -> parser.parse("CREATE EXTERNAL TABLE a.t (id int) STORED AS PARQUET LOCATION 'http://b/x'"));+ assertThat(location.sqlState(), equalTo("22023"));+ }++ @Test+ void rejectsFormatsOutsideTheSupportedAthenaReadPlan() {+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () -> parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (id int) STORED AS ORC LOCATION 's3://bucket/events/'"));+ assertThat(error.sqlState(), equalTo("0A000"));+ }++ @Test+ void dropStatementsDefaultToRestrictAndOnlyCascadeWhenAsked() {+ assertThat(parser.parse("DROP SCHEMA analytics").orElseThrow(),+ equalTo(new ExternalStatement.DropSchema("analytics", false, false)));+ assertThat(parser.parse("DROP SCHEMA analytics RESTRICT").orElseThrow(),+ equalTo(new ExternalStatement.DropSchema("analytics", false, false)));+ assertThat(parser.parse("DROP SCHEMA IF EXISTS analytics CASCADE;").orElseThrow(),+ equalTo(new ExternalStatement.DropSchema("analytics", true, true)));+ assertThat(parser.parse("DROP TABLE analytics.events").orElseThrow(),+ equalTo(new ExternalStatement.DropTable("analytics", "events", false, false)));+ assertThat(parser.parse("DROP TABLE IF EXISTS analytics.events RESTRICT").orElseThrow(),+ equalTo(new ExternalStatement.DropTable("analytics", "events", true, false)));+ assertThat(parser.parse("drop table analytics.events cascade").orElseThrow(),+ equalTo(new ExternalStatement.DropTable("analytics", "events", false, true)));+ }++ @Test+ void parsesCreateExternalSchemaIfNotExists() {+ ExternalStatement.CreateSchema schema = (ExternalStatement.CreateSchema) parser.parse(+ "CREATE EXTERNAL SCHEMA IF NOT EXISTS Analytics FROM DATA CATALOG DATABASE 'lake' IAM_ROLE 'arn:aws:iam::000000000000:role/R'").orElseThrow();+ assertThat(schema.schemaName(), equalTo("analytics"));+ assertThat(schema.ifNotExists(), equalTo(true));+ ExternalStatement.CreateSchema plain = (ExternalStatement.CreateSchema) parser.parse(+ "CREATE EXTERNAL SCHEMA analytics FROM DATA CATALOG DATABASE 'lake' IAM_ROLE 'arn:aws:iam::000000000000:role/R'").orElseThrow();+ assertThat(plain.ifNotExists(), equalTo(false));+ }++ @Test+ void canonicalisesRedshiftTypeAliasesToHiveTypes() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (a DOUBLE PRECISION, b int8, c float4, d bool, e character varying(10)) "+ + "STORED AS PARQUET LOCATION 's3://bucket/events/'").orElseThrow();+ assertThat(table.columns().stream().map(ExternalStatement.ColumnDefinition::type).toList(),+ equalTo(List.of("double", "bigint", "float", "boolean", "varchar(10)")));+ }++ @Test+ void parsesTablePropertiesWithParenthesesAndEscapedQuotes() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (id BIGINT) STORED AS PARQUET LOCATION 's3://bucket/events/' "+ + "TABLE PROPERTIES ('comment'='a (b) it''s', 'numRows'='5')").orElseThrow();+ assertThat(table.properties().get("comment"), equalTo("a (b) it's"));+ assertThat(table.properties().get("numRows"), equalTo("5"));+ }++ @Test+ void parsesPartitionValuesContainingClosingParentheses() {+ ExternalStatement.AddPartitions add = (ExternalStatement.AddPartitions) parser.parse(+ "ALTER TABLE analytics.events ADD PARTITION (region='eu (west)') LOCATION 's3://bucket/events/eu/'").orElseThrow();+ assertThat(add.partitions().get(0).values().get("region"), equalTo("eu (west)"));+ assertThat(add.partitions().get(0).location(), equalTo("s3://bucket/events/eu/"));+ }++ @Test+ void parsesStatementsThatStartWithComments() {+ assertThat(parser.parse("-- drop it\n/* now */ DROP TABLE analytics.events").orElseThrow(),+ equalTo(new ExternalStatement.DropTable("analytics", "events", false, false)));+ }++ @Test+ void parsesQuotedNamesContainingDotsAndSpaces() {+ assertThat(parser.parse("DROP TABLE \"my.schema\".\"my table\"").orElseThrow(),+ equalTo(new ExternalStatement.DropTable("my.schema", "my table", false, false)));+ assertThat(parser.parse("DROP SCHEMA \"a b\" CASCADE").orElseThrow(),+ equalTo(new ExternalStatement.DropSchema("a b", false, true)));+ }++ @Test+ void rejectsIamRoleThatIsNotOneQuotedLiteral() {+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class, () -> parser.parse(+ "CREATE EXTERNAL SCHEMA a FROM DATA CATALOG DATABASE 'lake' IAM_ROLE 'arn'; DROP TABLE x"));+ assertThat(exception.sqlState(), equalTo("42601"));+ }++ @Test+ void reportsMissingStoredAsClearly() {+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class, () -> parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (id BIGINT) LOCATION 's3://bucket/events/'"));+ assertThat(exception.getMessage(), equalTo("external table requires a STORED AS clause"));+ }++ @Test+ void keepsPropertyOrderAsWritten() {+ ExternalStatement.CreateTable table = (ExternalStatement.CreateTable) parser.parse(+ "CREATE EXTERNAL TABLE analytics.events (id BIGINT) STORED AS PARQUET LOCATION 's3://bucket/events/' "+ + "TABLE PROPERTIES ('z'='1', 'a'='2', 'm'='3')").orElseThrow();+ assertThat(List.copyOf(table.properties().keySet()), equalTo(List.of("z", "a", "m")));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilderTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilderTest.javanew file mode 100644index 0000000000..7ad498eb70--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTableBuilderTest.java@@ -0,0 +1,97 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import io.github.hectorvent.floci.services.glue.model.Partition;+import io.github.hectorvent.floci.services.glue.model.Table;+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatement.ColumnDefinition;+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatement.CreateTable;+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatement.PartitionSpec;+import io.github.hectorvent.floci.services.redshift.spectrum.ExternalStatement.TableFormat;+import org.junit.jupiter.api.Test;++import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertThrows;++class GlueTableBuilderTest {+ @Test+ void buildsGlueStorageDescriptorForParquetAndCsv() {+ CreateTable parquet = new CreateTable("analytics", "events",+ List.of(new ColumnDefinition("id", "int")), List.of(), TableFormat.PARQUET,+ "s3://bucket/events/", null, null, Map.of());+ Table table = GlueTableBuilder.toGlueTable(parquet);+ assertThat(table.getTableType(), equalTo("EXTERNAL_TABLE"));+ assertThat(table.getStorageDescriptor().getInputFormat(),+ equalTo("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"));+ assertThat(table.getParameters().get("EXTERNAL"), equalTo("TRUE"));++ CreateTable csv = new CreateTable("analytics", "csv_events",+ List.of(new ColumnDefinition("id", "int")), List.of(), TableFormat.TEXTFILE,+ "s3://bucket/csv/", "|", null, Map.of());+ assertThat(GlueTableBuilder.toGlueTable(csv).getStorageDescriptor().getSerdeInfo()+ .getParameters().get("field.delim"), equalTo("|"));+ }++ @Test+ void buildsPartitionValuesInGlueKeyOrder() {+ CreateTable statement = new CreateTable("analytics", "events",+ List.of(new ColumnDefinition("id", "int")),+ List.of(new ColumnDefinition("dt", "string"), new ColumnDefinition("region", "string")),+ TableFormat.PARQUET, "s3://bucket/events/", null, null, Map.of());+ Table table = GlueTableBuilder.toGlueTable(statement);+ Map<String, String> values = new LinkedHashMap<>();+ values.put("region", "eu");+ values.put("dt", "2024-01-01");++ Partition partition = GlueTableBuilder.toGluePartition(table,+ new PartitionSpec(values, "s3://bucket/events/dt=2024-01-01/region=eu/"));+ assertThat(partition.getValues(), equalTo(List.of("2024-01-01", "eu")));+ assertThat(partition.getStorageDescriptor().getLocation(),+ equalTo("s3://bucket/events/dt=2024-01-01/region=eu/"));+ }++ @Test+ void openCsvTableKeepsItsSerdePropertiesAndNoDefaultFieldDelimiter() {+ CreateTable statement = new CreateTable("analytics", "events",+ List.of(new ColumnDefinition("id", "int")), List.of(), TableFormat.TEXTFILE,+ "s3://bucket/events/", null, "org.apache.hadoop.hive.serde2.OpenCSVSerde", Map.of(),+ Map.of("separatorChar", "|", "quoteChar", "\""));++ Map<String, String> parameters = GlueTableBuilder.toGlueTable(statement)+ .getStorageDescriptor().getSerdeInfo().getParameters();++ assertThat(parameters.get("separatorChar"), equalTo("|"));+ assertThat(parameters.get("quoteChar"), equalTo("\""));+ assertThat(parameters.containsKey("field.delim"), equalTo(false));+ assertThat(GlueTableBuilder.toGlueTable(statement).getStorageDescriptor().getSerdeInfo().getSerializationLibrary(),+ equalTo("org.apache.hadoop.hive.serde2.OpenCSVSerde"));+ }++ @Test+ void rejectsPartitionLocationsOutsideS3AndKeysTheTableDoesNotDeclare() {+ Table table = GlueTableBuilder.toGlueTable(new CreateTable("analytics", "events", List.of(),+ List.of(new ColumnDefinition("dt", "string")), TableFormat.PARQUET,+ "s3://bucket/events/", null, null, Map.of()));++ SpectrumSqlException location = assertThrows(SpectrumSqlException.class, () ->+ GlueTableBuilder.toGluePartition(table, new PartitionSpec(Map.of("dt", "x"), "/local/path/")));+ SpectrumSqlException extraKey = assertThrows(SpectrumSqlException.class, () ->+ GlueTableBuilder.toGluePartition(table, new PartitionSpec(Map.of("dt", "x", "extra", "y"), "s3://b/x/")));++ assertThat(location.sqlState(), equalTo("22023"));+ assertThat(extraKey.sqlState(), equalTo("22023"));+ }++ @Test+ void rejectsMissingPartitionKeys() {+ Table table = GlueTableBuilder.toGlueTable(new CreateTable("analytics", "events", List.of(),+ List.of(new ColumnDefinition("dt", "string")), TableFormat.PARQUET,+ "s3://bucket/events/", null, null, Map.of()));+ SpectrumSqlException error = assertThrows(SpectrumSqlException.class, () ->+ GlueTableBuilder.toGluePartition(table, new PartitionSpec(Map.of("other", "x"), "s3://b/x/")));+ assertThat(error.sqlState(), equalTo("22023"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapperTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapperTest.javanew file mode 100644index 0000000000..916b07d884--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/GlueTypeMapperTest.java@@ -0,0 +1,65 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import org.junit.jupiter.api.Test;++import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.is;+import static org.junit.jupiter.api.Assertions.assertThrows;++class GlueTypeMapperTest {+ @Test+ void mapsGlueScalarAndParameterizedTypes() {+ assertThat(GlueTypeMapper.toPostgres("string"), equalTo("text"));+ assertThat(GlueTypeMapper.toPostgres("int"), equalTo("integer"));+ assertThat(GlueTypeMapper.toPostgres("DECIMAL(10, 2)"), equalTo("numeric(10,2)"));+ assertThat(GlueTypeMapper.toPostgres("char(3)"), equalTo("varchar(3)"));+ assertThat(GlueTypeMapper.toPostgres("character(3)"), equalTo("varchar(3)"));+ assertThat(GlueTypeMapper.toPostgres("varbyte"), equalTo("bytea"));+ assertThat(GlueTypeMapper.toPostgres("binary"), equalTo("bytea"));+ assertThat(GlueTypeMapper.toPostgres("array<string>"), equalTo("jsonb"));+ assertThat(GlueTypeMapper.isNested("array<string>"), is(true));+ }++ @Test+ void rejectsUnknownType() {+ SpectrumSqlException exception = assertThrows(SpectrumSqlException.class,+ () -> GlueTypeMapper.toPostgres("interval"));+ assertThat(exception.sqlState(), equalTo("0A000"));+ }++ @Test+ void quotesProjectionIdentifiersAndPreservesNullMarker() {+ assertThat(GlueTypeMapper.duckProjection("we\"ird", "string"),+ equalTo("COALESCE(CAST(\"we\"\"ird\" AS VARCHAR), '\\N') AS \"we\"\"ird\""));+ assertThat(GlueTypeMapper.duckProjection("payload", "varbyte"),+ equalTo("COALESCE(CAST('\\x' || hex(\"payload\") AS VARCHAR), '\\N') AS \"payload\""));+ assertThat(GlueTypeMapper.duckProjection("payload", "binary"),+ equalTo("COALESCE(CAST('\\x' || hex(\"payload\") AS VARCHAR), '\\N') AS \"payload\""));+ }++ @Test+ void canonicalisesRedshiftAliases() {+ assertThat(GlueTypeMapper.canonicalGlueType("DOUBLE PRECISION"), equalTo("double"));+ assertThat(GlueTypeMapper.canonicalGlueType("float8"), equalTo("double"));+ assertThat(GlueTypeMapper.canonicalGlueType("real"), equalTo("float"));+ assertThat(GlueTypeMapper.canonicalGlueType("int2"), equalTo("smallint"));+ assertThat(GlueTypeMapper.canonicalGlueType("int4"), equalTo("int"));+ assertThat(GlueTypeMapper.canonicalGlueType("INT8"), equalTo("bigint"));+ assertThat(GlueTypeMapper.canonicalGlueType("bool"), equalTo("boolean"));+ assertThat(GlueTypeMapper.canonicalGlueType("character varying(20)"), equalTo("varchar(20)"));+ assertThat(GlueTypeMapper.canonicalGlueType("CHARACTER VARYING"), equalTo("varchar(256)"));+ assertThat(GlueTypeMapper.canonicalGlueType("VARCHAR"), equalTo("varchar(256)"));+ assertThat(GlueTypeMapper.canonicalGlueType("CHARACTER(10)"), equalTo("char(10)"));+ assertThat(GlueTypeMapper.canonicalGlueType("character"), equalTo("char(1)"));+ assertThat(GlueTypeMapper.canonicalGlueType("CHAR"), equalTo("char(1)"));+ assertThat(GlueTypeMapper.canonicalGlueType("VARBYTE"), equalTo("binary"));+ assertThat(GlueTypeMapper.canonicalGlueType("varbyte(64000)"), equalTo("binary"));+ assertThat(GlueTypeMapper.canonicalGlueType("VARBINARY"), equalTo("binary"));+ assertThat(GlueTypeMapper.canonicalGlueType("binary varying(100)"), equalTo("binary"));+ assertThat(GlueTypeMapper.canonicalGlueType("Decimal(10, 2)"), equalTo("decimal(10,2)"));+ assertThat(GlueTypeMapper.canonicalGlueType("NUMERIC(10, 2)"), equalTo("decimal(10,2)"));+ assertThat(GlueTypeMapper.canonicalGlueType("numeric"), equalTo("decimal"));+ assertThat(GlueTypeMapper.canonicalGlueType("array<string>"), equalTo("array<string>"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.javanew file mode 100644index 0000000000..c0614bb9d4--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/PostgresBackendSessionTest.java@@ -0,0 +1,191 @@+package io.github.hectorvent.floci.services.redshift.spectrum;++import org.junit.jupiter.api.Test;++import java.io.ByteArrayInputStream;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.ServerSocket;+import java.net.Socket;+import java.nio.charset.StandardCharsets;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertThrows;++class PostgresBackendSessionTest {++ @Test+ void copyReadFailureDrainsCopyFailResponseWithoutReplacingOriginalCause() throws Exception {+ try (ServerSocket listener = new ServerSocket(0);+ Socket backend = new Socket("localhost", listener.getLocalPort())) {+ ExecutorService executor = Executors.newSingleThreadExecutor();+ Future<Character> copyFailMessage = executor.submit(() -> serveCopyFailure(listener));+ SpectrumReadException rowFailure = new SpectrumReadException("22000", "bad CSV row");+ InputStream failingInput = new InputStream() {+ @Override+ public int read() throws IOException {+ throw new IOException("reader failed", rowFailure);+ }+ };++ SpectrumReadException thrown = assertThrows(SpectrumReadException.class,+ () -> new PostgresBackendSession(backend).copyIn("COPY \"t\" FROM STDIN", failingInput));++ assertEquals("58030", thrown.sqlState());+ assertSame(rowFailure, thrown.getCause().getCause());+ assertEquals('f', copyFailMessage.get());+ executor.shutdownNow();+ }+ }++ @Test+ void executeSurfacesBackendErrorWithItsSqlState() throws Exception {+ try (ServerSocket listener = new ServerSocket(0);+ Socket backend = new Socket("localhost", listener.getLocalPort())) {+ ExecutorService executor = Executors.newSingleThreadExecutor();+ Future<Character> queryMessage = executor.submit(() -> {+ try (Socket socket = listener.accept()) {+ Frame query = readFrame(socket.getInputStream());+ send(socket.getOutputStream(), 'E', new byte[]{'C', '4', '2', 'P', '0', '1', 0,+ 'M', 'n', 'o', ' ', 't', 'a', 'b', 'l', 'e', 0, 0});+ send(socket.getOutputStream(), 'Z', new byte[]{'I'});+ return query.type();+ }+ });++ SpectrumReadException thrown = assertThrows(SpectrumReadException.class,+ () -> new PostgresBackendSession(backend).execute("SELECT 1"));++ assertEquals("42P01", thrown.sqlState());+ assertEquals('Q', queryMessage.get());+ executor.shutdownNow();+ }+ }++ @Test+ void copyInStreamsDataAndReturnsTheRowCountFromTheCommandTag() throws Exception {+ try (ServerSocket listener = new ServerSocket(0);+ Socket backend = new Socket("localhost", listener.getLocalPort())) {+ ExecutorService executor = Executors.newSingleThreadExecutor();+ Future<String> received = executor.submit(() -> {+ try (Socket socket = listener.accept()) {+ InputStream input = socket.getInputStream();+ OutputStream output = socket.getOutputStream();+ readFrame(input);+ send(output, 'G', new byte[]{0, 0, 0});+ StringBuilder data = new StringBuilder();+ Frame frame = readFrame(input);+ while (frame.type() == 'd') {+ data.append(new String(frame.body(), StandardCharsets.UTF_8));+ frame = readFrame(input);+ }+ send(output, 'C', "COPY 3\0".getBytes(StandardCharsets.UTF_8));+ send(output, 'Z', new byte[]{'I'});+ return data + "|" + frame.type();+ }+ });++ long rows = new PostgresBackendSession(backend).copyIn("COPY \"t\" FROM STDIN",+ new ByteArrayInputStream("a\nb\nc\n".getBytes(StandardCharsets.UTF_8)));++ assertEquals(3, rows);+ assertEquals("a\nb\nc\n|c", received.get());+ executor.shutdownNow();+ }+ }++ @Test+ void copyInContinuesAfterAZeroByteRead() throws Exception {+ try (ServerSocket listener = new ServerSocket(0);+ Socket backend = new Socket("localhost", listener.getLocalPort())) {+ ExecutorService executor = Executors.newSingleThreadExecutor();+ Future<String> received = executor.submit(() -> {+ try (Socket socket = listener.accept()) {+ InputStream input = socket.getInputStream();+ OutputStream output = socket.getOutputStream();+ readFrame(input);+ send(output, 'G', new byte[]{0, 0, 0});+ StringBuilder data = new StringBuilder();+ Frame frame = readFrame(input);+ while (frame.type() == 'd') {+ data.append(new String(frame.body(), StandardCharsets.UTF_8));+ frame = readFrame(input);+ }+ send(output, 'C', "COPY 1\0".getBytes(StandardCharsets.UTF_8));+ send(output, 'Z', new byte[]{'I'});+ return data + "|" + frame.type();+ }+ });+ ByteArrayInputStream source = new ByteArrayInputStream("row\n".getBytes(StandardCharsets.UTF_8));+ InputStream zeroByteReadInput = new InputStream() {+ private boolean returnZero = true;++ @Override+ public int read() {+ return source.read();+ }++ @Override+ public int read(byte[] bytes, int offset, int length) {+ if (returnZero) {+ returnZero = false;+ return 0;+ }+ return source.read(bytes, offset, length);+ }+ };++ long rows = new PostgresBackendSession(backend).copyIn("COPY \"t\" FROM STDIN", zeroByteReadInput);++ assertEquals(1, rows);+ assertEquals("row\n|c", received.get());+ executor.shutdownNow();+ }+ }+++ private static Character serveCopyFailure(ServerSocket listener) throws IOException {+ try (Socket socket = listener.accept()) {+ InputStream input = socket.getInputStream();+ OutputStream output = socket.getOutputStream();+ readFrame(input);+ send(output, 'G', new byte[]{0, 0, 0});+ Frame copyFail = readFrame(input);+ byte[] error = new byte[]{'C', '5', '7', '0', '1', '4', 0, 'M', 'C', 'O', 'P', 'Y', ' ', 'f',+ 'a', 'i', 'l', 'e', 'd', 0, 0};+ send(output, 'E', error);+ send(output, 'Z', new byte[]{'I'});+ return copyFail.type();+ }+ }++ private static Frame readFrame(InputStream input) throws IOException {+ int type = input.read();+ if (type < 0) {+ throw new IOException("unexpected EOF");+ }+ byte[] length = input.readNBytes(4);+ int size = ((length[0] & 0xff) << 24) | ((length[1] & 0xff) << 16)+ | ((length[2] & 0xff) << 8) | (length[3] & 0xff);+ return new Frame((char) type, input.readNBytes(size - 4));+ }++ private static void send(OutputStream output, char type, byte[] body) throws IOException {+ int size = body.length + 4;+ output.write(type);+ output.write((size >>> 24) & 0xff);+ output.write((size >>> 16) & 0xff);+ output.write((size >>> 8) & 0xff);+ output.write(size & 0xff);+ output.write(body);+ output.flush();+ }++ private record Frame(char type, byte[] body) {+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.javaindex 9c633066e7..b2a690d6c0 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/spectrum/SpectrumMaterializerTest.java@@ -9,8 +9,8 @@ import java.io.OutputStream;import java.net.ServerSocket;import java.net.Socket;import java.nio.charset.StandardCharsets;-import java.util.List;import java.util.Arrays;+import java.util.List;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;@@ -24,13 +24,17 @@ import static org.mockito.Mockito.when;class SpectrumMaterializerTest {@Test- void materializesRowsWithGeneratedIdentifierAndCopyFraming() throws Exception {+ void materializesRowsAndSetsSurplusVarcharValuesToNull() throws Exception {try (ServerSocket listener = new ServerSocket(0); Socket backend = new Socket("localhost", listener.getLocalPort())) {ExecutorService executor = Executors.newSingleThreadExecutor();- Future<byte[]> captured = executor.submit(() -> serve(listener));+ Future<BackendCapture> captured = executor.submit(() -> serve(listener));SpectrumS3Reader reader = mock(SpectrumS3Reader.class);when(reader.read(org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.any()))- .thenReturn(Stream.of(new SpectrumRow(Arrays.asList("1", null)), new SpectrumRow(List.of("two", "x"))));+ .thenReturn(Stream.of(new SpectrumRow(Arrays.asList("1", null)),+ new SpectrumRow(List.of("two", "x")),+ new SpectrumRow(List.of("3", "v".repeat(256))),+ new SpectrumRow(List.of("4", "v".repeat(257))),+ new SpectrumRow(List.of("5", "é".repeat(129)))));SpectrumExternalTable table = new SpectrumExternalTable("000000000000", "dev", "analytics", "events",List.of(new SpectrumColumn("id", SpectrumColumn.Type.INTEGER),new SpectrumColumn("value", SpectrumColumn.Type.VARCHAR)),@@ -40,20 +44,23 @@ class SpectrumMaterializerTest {.materialize(backend, table, null, reader);new SpectrumMaterializer().cleanup(backend, materialization);- byte[] copy = captured.get();+ BackendCapture capture = captured.get();assertTrue(materialization.identifier().startsWith("spectrum_tmp_"));assertFalse(materialization.identifier().contains("events"));- assertTrue(new String(copy, StandardCharsets.UTF_8).contains("1\t\\N\ntwo\tx\n"));+ assertTrue(capture.createSql().contains("\"value\" VARCHAR(256)"));+ String copyRows = new String(capture.copyRows(), StandardCharsets.UTF_8);+ assertTrue(copyRows.contains("1\t\\N\ntwo\tx\n"));+ assertTrue(copyRows.contains("3\t" + "v".repeat(256) + "\n4\t\\N\n5\t\\N\n"));executor.shutdownNow();}}- private static byte[] serve(ServerSocket listener) throws IOException {+ private static BackendCapture serve(ServerSocket listener) throws IOException {try (Socket socket = listener.accept()) {InputStream input = socket.getInputStream();OutputStream output = socket.getOutputStream();ByteArrayOutputStream copyRows = new ByteArrayOutputStream();- readQuery(input);+ String createSql = readQuery(input);send(output, 'C', "CREATE 0\0".getBytes(StandardCharsets.UTF_8));send(output, 'Z', new byte[]{'I'});readQuery(input);@@ -72,15 +79,16 @@ class SpectrumMaterializerTest {readQuery(input);send(output, 'C', "DROP TABLE\0".getBytes(StandardCharsets.UTF_8));send(output, 'Z', new byte[]{'I'});- return copyRows.toByteArray();+ return new BackendCapture(createSql, copyRows.toByteArray());}}- private static void readQuery(InputStream input) throws IOException {+ private static String readQuery(InputStream input) throws IOException {Frame frame = readFrame(input);if (frame.type() != 'Q') {throw new IOException("expected query");}+ return new String(frame.body(), 0, frame.body().length - 1, StandardCharsets.UTF_8);}private static Frame readFrame(InputStream input) throws IOException {@@ -107,4 +115,7 @@ class SpectrumMaterializerTest {private record Frame(char type, byte[] body) {}++ private record BackendCapture(String createSql, byte[] copyRows) {+ }}