Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(athena): register every Glue database as a DuckDB schema so <db>.<table> resolves (#2993)

Vũ Thế Huy0ab9979edf30
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java
index 0e52929408..4def372ded 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java
@@ -1,12 +1,15 @@
package com.floci.test;
import org.junit.jupiter.api.*;
+import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.athena.AthenaClient;
import software.amazon.awssdk.services.athena.model.*;
import software.amazon.awssdk.services.glue.GlueClient;
+import software.amazon.awssdk.services.s3.S3Client;
import java.time.Duration;
import java.time.Instant;
+import java.util.Map;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
@@ -152,22 +155,30 @@ class AthenaTest {
)
);
- GetTableMetadataResponse response = athena.getTableMetadata(
- GetTableMetadataRequest.builder()
- .catalogName("AwsDataCatalog")
- .databaseName(dbName)
- .tableName(tableName)
- .build()
- );
+ try {
+ GetTableMetadataResponse response = athena.getTableMetadata(
+ GetTableMetadataRequest.builder()
+ .catalogName("AwsDataCatalog")
+ .databaseName(dbName)
+ .tableName(tableName)
+ .build()
+ );
- assertThat(response.tableMetadata().createTime())
- .as("createTime must be parseable by the AWS SDK")
- .isNotNull();
- assertThat(response.tableMetadata().lastAccessTime())
- .as("lastAccessTime must be parseable by the AWS SDK")
- .isNotNull();
-
- glue.close();
+ assertThat(response.tableMetadata().createTime())
+ .as("createTime must be parseable by the AWS SDK")
+ .isNotNull();
+ assertThat(response.tableMetadata().lastAccessTime())
+ .as("lastAccessTime must be parseable by the AWS SDK")
+ .isNotNull();
+ } finally {
+ try {
+ glue.deleteTable(r -> r.databaseName(dbName).name(tableName));
+ glue.deleteDatabase(r -> r.name(dbName));
+ } catch (Exception ignored) {
+ // Best-effort test cleanup
+ }
+ glue.close();
+ }
}
@Test
@@ -178,31 +189,141 @@ class AthenaTest {
String location = "s3://test-bucket/" + dbName + "/";
try (GlueClient glue = TestFixtures.glueClient()) {
- StartQueryExecutionResponse started = athena.startQueryExecution(
- StartQueryExecutionRequest.builder()
- .queryString("CREATE DATABASE IF NOT EXISTS " + dbName
- + " LOCATION '" + location + "'")
- .workGroup("primary")
- .build());
-
- QueryExecutionStatus status = TestFixtures.awaitAthenaQueryTerminal(
- athena, started.queryExecutionId(), Duration.ofSeconds(60));
- assertThat(status.state())
- .as("Athena DDL did not succeed: %s", status.stateChangeReason())
- .isEqualTo(QueryExecutionState.SUCCEEDED);
-
- QueryExecution execution = athena.getQueryExecution(r -> r
- .queryExecutionId(started.queryExecutionId())).queryExecution();
- assertThat(execution.statementType()).isEqualTo(StatementType.DDL);
- assertThat(execution.resultConfiguration()).isNull();
-
- GetQueryResultsResponse results = athena.getQueryResults(r -> r
- .queryExecutionId(started.queryExecutionId()));
- assertThat(results.resultSet().rows()).isEmpty();
- assertThat(results.resultSet().resultSetMetadata().columnInfo()).isEmpty();
-
- assertThat(glue.getDatabase(r -> r.name(dbName)).database().locationUri())
- .isEqualTo(location);
+ try {
+ StartQueryExecutionResponse started = athena.startQueryExecution(
+ StartQueryExecutionRequest.builder()
+ .queryString("CREATE DATABASE IF NOT EXISTS " + dbName
+ + " LOCATION '" + location + "'")
+ .workGroup("primary")
+ .build());
+
+ QueryExecutionStatus status = TestFixtures.awaitAthenaQueryTerminal(
+ athena, started.queryExecutionId(), Duration.ofSeconds(60));
+ assertThat(status.state())
+ .as("Athena DDL did not succeed: %s", status.stateChangeReason())
+ .isEqualTo(QueryExecutionState.SUCCEEDED);
+
+ QueryExecution execution = athena.getQueryExecution(r -> r
+ .queryExecutionId(started.queryExecutionId())).queryExecution();
+ assertThat(execution.statementType()).isEqualTo(StatementType.DDL);
+ assertThat(execution.resultConfiguration()).isNull();
+
+ GetQueryResultsResponse results = athena.getQueryResults(r -> r
+ .queryExecutionId(started.queryExecutionId()));
+ assertThat(results.resultSet().rows()).isEmpty();
+ assertThat(results.resultSet().resultSetMetadata().columnInfo()).isEmpty();
+
+ assertThat(glue.getDatabase(r -> r.name(dbName)).database().locationUri())
+ .isEqualTo(location);
+ } finally {
+ try {
+ glue.deleteDatabase(r -> r.name(dbName));
+ } catch (Exception ignored) {
+ // Best-effort test cleanup
+ }
+ }
+ }
+ }
+
+ /**
+ * Reproduces issue #2859: Athena queries referencing tables with qualified names (e.g.
+ * {@code FROM shop.orders}) failed with "schema does not exist" when DuckDB schemas were not
+ * registered for Glue databases.
+ */
+ @Test
+ @Order(9)
+ @DisplayName("Query table with qualified database schema resolves and executes")
+ void queryTableWithQualifiedDatabaseSchemaResolves() throws InterruptedException {
+ String uniqueSuffix = UUID.randomUUID().toString().replace("-", "").substring(0, 8);
+ String dbName = "shop_" + uniqueSuffix;
+ String tableName = "orders";
+ String bucketName = "athena-compat-" + uniqueSuffix;
+ String dataPrefix = "orders/";
+ String location = "s3://" + bucketName + "/" + dataPrefix;
+
+ try (GlueClient glue = TestFixtures.glueClient();
+ S3Client s3 = TestFixtures.s3Client()) {
+
+ s3.createBucket(r -> r.bucket(bucketName));
+ s3.putObject(r -> r.bucket(bucketName).key(dataPrefix + "orders.json"),
+ RequestBody.fromString(
+ "{\"customer\":\"ana\",\"amount\":10}\n{\"customer\":\"leo\",\"amount\":7}\n"));
+
+ glue.createDatabase(r -> r.databaseInput(i -> i.name(dbName)));
+ glue.createTable(r -> r
+ .databaseName(dbName)
+ .tableInput(t -> t
+ .name(tableName)
+ .tableType("EXTERNAL_TABLE")
+ .storageDescriptor(sd -> sd
+ .location(location)
+ .inputFormat("org.apache.hadoop.mapred.TextInputFormat")
+ .outputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat")
+ .serdeInfo(s -> s
+ .serializationLibrary("org.openx.data.jsonserde.JsonSerDe")
+ .parameters(Map.of("serialization.format", "1")))
+ .columns(
+ software.amazon.awssdk.services.glue.model.Column.builder().name("customer").type("string").build(),
+ software.amazon.awssdk.services.glue.model.Column.builder().name("amount").type("int").build()
+ )
+ )
+ )
+ );
+
+ try {
+ // Qualified query without context database (issue #2859 exact shape)
+ StartQueryExecutionResponse started = athena.startQueryExecution(
+ StartQueryExecutionRequest.builder()
+ .queryString("SELECT customer, sum(amount) as total FROM " + dbName + "." + tableName
+ + " GROUP BY customer ORDER BY customer")
+ .workGroup("primary")
+ .resultConfiguration(ResultConfiguration.builder()
+ .outputLocation("s3://" + bucketName + "/results/")
+ .build())
+ .build());
+
+ QueryExecutionStatus status = TestFixtures.awaitAthenaQueryTerminal(
+ athena, started.queryExecutionId(), Duration.ofSeconds(60));
+ assertThat(status.state())
+ .as("Athena qualified query did not succeed: %s", status.stateChangeReason())
+ .isEqualTo(QueryExecutionState.SUCCEEDED);
+
+ GetQueryResultsResponse results = athena.getQueryResults(r -> r
+ .queryExecutionId(started.queryExecutionId()));
+ assertThat(results.resultSet().rows()).hasSizeGreaterThanOrEqualTo(3);
+
+ // Unqualified query with context database (context alias verification)
+ StartQueryExecutionResponse startedUnqualified = athena.startQueryExecution(
+ StartQueryExecutionRequest.builder()
+ .queryString("SELECT customer, sum(amount) as total FROM " + tableName
+ + " GROUP BY customer ORDER BY customer")
+ .queryExecutionContext(QueryExecutionContext.builder().database(dbName).build())
+ .workGroup("primary")
+ .resultConfiguration(ResultConfiguration.builder()
+ .outputLocation("s3://" + bucketName + "/results/")
+ .build())
+ .build());
+
+ QueryExecutionStatus statusUnqualified = TestFixtures.awaitAthenaQueryTerminal(
+ athena, startedUnqualified.queryExecutionId(), Duration.ofSeconds(60));
+ assertThat(statusUnqualified.state())
+ .as("Athena unqualified query did not succeed: %s", statusUnqualified.stateChangeReason())
+ .isEqualTo(QueryExecutionState.SUCCEEDED);
+ } finally {
+ try {
+ glue.deleteTable(r -> r.databaseName(dbName).name(tableName));
+ glue.deleteDatabase(r -> r.name(dbName));
+ } catch (Exception ignored) {
+ // Best-effort test cleanup
+ }
+ try {
+ s3.listObjectsV2(r -> r.bucket(bucketName)).contents().forEach(obj ->
+ s3.deleteObject(r -> r.bucket(bucketName).key(obj.key())));
+ s3.deleteBucket(r -> r.bucket(bucketName));
+ } catch (Exception ignored) {
+ // Best-effort test cleanup
+ }
+ }
}
}
}
diff --git a/docs/services/athena.md b/docs/services/athena.md
index 48da8fdca6..a80abfa8b3 100644
--- a/docs/services/athena.md
+++ b/docs/services/athena.md
@@ -3,7 +3,7 @@
**Protocol:** JSON 1.1
**Endpoint:** `http://localhost:4566/`
-Floci emulates Amazon Athena with **real SQL execution** powered by a [floci-duck](https://hub.docker.com/r/floci/floci-duck) sidecar container running DuckDB. When a query is submitted, Floci spins up the sidecar on first use, injects `CREATE OR REPLACE VIEW` statements for each Glue-registered table pointing to S3 data, then executes the SQL and stores results as CSV in S3.
+Floci emulates Amazon Athena with **real SQL execution** powered by a [floci-duck](https://hub.docker.com/r/floci/floci-duck) sidecar container running DuckDB. When a query is submitted, Floci spins up the sidecar on first use, registers every Glue database as a DuckDB schema (`CREATE SCHEMA IF NOT EXISTS "<schema>"`) with qualified views (`"<db>"."<table>"`) pointing to S3 data, creates unqualified view aliases for the query context database in DuckDB's default `main` schema, then executes the SQL and stores results as CSV in S3.
## Supported Actions
@@ -29,7 +29,7 @@ Floci emulates Amazon Athena with **real SQL execution** powered by a [floci-duc
## How it works
1. **Lazy sidecar start**: On the first `StartQueryExecution` call, Floci checks for a local `floci/floci-duck:latest` image and starts the container. Subsequent queries reuse the running container.
-2. **Glue DDL injection**: Floci reads all Glue tables for the target database and generates `CREATE OR REPLACE VIEW` statements mapping each table name to its S3 location via DuckDB's `read_parquet`, `read_json_auto`, or `read_csv_auto` functions — chosen based on the table's `InputFormat` or SerDe serialization library.
+2. **Glue DDL injection**: Floci registers every Glue database as a DuckDB schema (`CREATE SCHEMA IF NOT EXISTS "<schema>"`) and creates qualified views (`CREATE OR REPLACE VIEW "<db>"."<table>"`) mapping each table to its S3 location via DuckDB's `read_parquet`, `read_json_auto`, or `read_csv_auto` functions, chosen based on the table's `InputFormat` or SerDe serialization library. In addition, tables in the query context database receive unqualified view aliases (`CREATE OR REPLACE VIEW "<table>"` in DuckDB's default `main` schema), so queries can reference tables as both `FROM db.table` and `FROM table`.
3. **Query execution**: The user's SQL is wrapped in `COPY (...) TO 's3://...' (FORMAT CSV, HEADER)` and executed. Results are written directly to the output S3 path.
4. **Results retrieval**: `GetQueryResults` reads the CSV back from S3 and returns it in the standard Athena `ResultSet` shape.
@@ -48,11 +48,11 @@ The DuckDB read function is chosen from the Glue table's `StorageDescriptor`:
| Property | Default | Description |
|---|---|---|
-| `FLOCI_SERVICES_ATHENA_MOCK` | `false` | Set to `true` to disable DuckDB execution — queries immediately succeed with empty results |
+| `FLOCI_SERVICES_ATHENA_MOCK` | `false` | Set to `true` to disable DuckDB execution: queries immediately succeed with empty results |
| `FLOCI_SERVICES_DUCK_DEFAULT_IMAGE` | `floci/floci-duck:latest` | DuckDB sidecar image pulled on first use |
| `FLOCI_SERVICES_DUCK_URL` | *(unset)* | Point to an existing floci-duck instance and skip container management |
-## Example — simple query
+## Example: simple query
```bash
export AWS_ENDPOINT_URL=http://localhost:4566
@@ -70,7 +70,7 @@ aws athena get-query-execution --query-execution-id $QUERY_ID
aws athena get-query-results --query-execution-id $QUERY_ID
```
-## Example — data lake query (S3 + Glue + Athena)
+## Example: data lake query (S3 + Glue + Athena)
```bash
export AWS_ENDPOINT_URL=http://localhost:4566
@@ -126,12 +126,12 @@ aws athena get-query-results --query-execution-id $QUERY_ID
## Shared sidecar with S3 Select
-The floci-duck sidecar is shared between Athena and S3 Select. Once started by the first Athena query, it is also used by `SelectObjectContent` for CSV (with `FileHeaderInfo=USE`), JSON, and Parquet inputs. If Athena has not yet executed a query, S3 Select falls back to the built-in Java evaluator for CSV and JSON — Parquet always requires the sidecar.
+The floci-duck sidecar is shared between Athena and S3 Select. Once started by the first Athena query, it is also used by `SelectObjectContent` for CSV (with `FileHeaderInfo=USE`), JSON, and Parquet inputs. If Athena has not yet executed a query, S3 Select falls back to the built-in Java evaluator for CSV and JSON. Parquet always requires the sidecar.
See [S3 Select](s3.md#s3-select) for details on execution modes and supported SQL operators.
## Mock mode
-Set `FLOCI_SERVICES_ATHENA_MOCK=true` to skip DuckDB entirely for Athena. In this mode queries transition to `SUCCEEDED` immediately with an empty result set — useful for unit tests that only exercise the Athena state machine, not the query results.
+Set `FLOCI_SERVICES_ATHENA_MOCK=true` to skip DuckDB entirely for Athena. In this mode queries transition to `SUCCEEDED` immediately with an empty result set, useful for unit tests that only exercise the Athena state machine, not the query results.
When mock mode is enabled the sidecar does **not** start. S3 Select will use the Java evaluator for CSV and JSON. Parquet queries will fail unless `FLOCI_SERVICES_DUCK_URL` points to an already-running floci-duck instance.
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
index 0122d00073..077fb8dd2a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
@@ -61,6 +61,7 @@ public class AthenaService {
private final S3Service s3Service;
private final EmulatorConfig config;
private final Vertx vertx;
+ private final GlueViewDdlBuilder ddlBuilder;
@Inject
public AthenaService(StorageFactory storageFactory,
@@ -68,7 +69,8 @@ public class AthenaService {
GlueService glueService,
S3Service s3Service,
EmulatorConfig config,
- Vertx vertx) {
+ Vertx vertx,
+ GlueViewDdlBuilder ddlBuilder) {
this.queryStore = storageFactory.create("athena", "queries.json",
new TypeReference<>() {});
this.workGroupStore = storageFactory.create("athena", "workgroups.json",
@@ -78,6 +80,7 @@ public class AthenaService {
this.s3Service = s3Service;
this.config = config;
this.vertx = vertx;
+ this.ddlBuilder = ddlBuilder;
}
public String startQueryExecution(String query,
@@ -132,7 +135,7 @@ public class AthenaService {
// Submit async — caller gets the ID immediately while execution runs in background
vertx.executeBlocking(() -> {
- String setupDdl = buildGlueDdl(database);
+ String setupDdl = ddlBuilder.build(database);
if (outputLocation != null) {
ensureOutputBucket(outputLocation);
}
@@ -367,62 +370,6 @@ public class AthenaService {
String location, Map<String, String> properties) {
}
- private String buildGlueDdl(String database) {
- StringBuilder sb = new StringBuilder();
- try {
- List<Table> tables = glueService.getTables(database);
- for (Table table : tables) {
- String location = table.getStorageDescriptor() != null
- ? table.getStorageDescriptor().getLocation()
- : null;
- if (location == null || location.isBlank()) {
- continue;
- }
- String readFn = inferReadFunction(table);
- String normalizedLocation = location.endsWith("/")
- ? location.substring(0, location.length() - 1) : location;
- sb.append("CREATE OR REPLACE VIEW \"")
- .append(table.getName())
- .append("\" AS SELECT * FROM ")
- .append(readExpression(readFn, normalizedLocation))
- .append(";\n");
- }
- } catch (Exception e) {
- LOG.debugv("Could not inject Glue DDL for database {0}: {1}", database, e.getMessage());
- }
- return sb.toString();
- }
-
- private String readExpression(String readFn, String normalizedLocation) {
- String glob = normalizedLocation + "/**";
- if ("read_parquet".equals(readFn)) {
- return "read_parquet('" + glob + "', union_by_name = true)";
- }
- return readFn + "('" + glob + "')";
- }
-
- private String inferReadFunction(Table table) {
- if (table.getStorageDescriptor() == null) {
- return "read_csv_auto";
- }
- String format = table.getStorageDescriptor().getInputFormat();
- String serde = table.getStorageDescriptor().getSerdeInfo() != null
- ? table.getStorageDescriptor().getSerdeInfo().getSerializationLibrary()
- : null;
- if (containsIgnoreCase(format, "parquet") || containsIgnoreCase(serde, "parquet")) {
- return "read_parquet";
- }
- if (containsIgnoreCase(format, "json") || containsIgnoreCase(serde, "json")
- || containsIgnoreCase(format, "hive")) {
- return "read_json_auto";
- }
- return "read_csv_auto";
- }
-
- private static boolean containsIgnoreCase(String str, String sub) {
- return str != null && str.toLowerCase().contains(sub);
- }
-
private String resolveOutputLocation(ResultConfiguration rc, String queryId) {
String base = (rc != null && rc.getOutputLocation() != null && !rc.getOutputLocation().isBlank())
? rc.getOutputLocation()
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java b/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java
new file mode 100644
index 0000000000..31fc24fdf3
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java
@@ -0,0 +1,134 @@
+package io.github.hectorvent.floci.services.athena;
+
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Database;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.util.List;
+import java.util.Locale;
+
+@ApplicationScoped
+public class GlueViewDdlBuilder {
+
+ private static final Logger LOG = Logger.getLogger(GlueViewDdlBuilder.class);
+
+ private final GlueService glueService;
+
+ @Inject
+ public GlueViewDdlBuilder(GlueService glueService) {
+ this.glueService = glueService;
+ }
+
+ public String build(String contextDatabase) {
+ StringBuilder sb = new StringBuilder();
+ boolean contextDbHandled = false;
+ List<Database> databases = glueService.getDatabases();
+ if (databases != null) {
+ for (Database db : databases) {
+ if (db == null) {
+ continue;
+ }
+ String schema = db.getName();
+ if (schema == null || schema.isBlank()) {
+ continue;
+ }
+ sb.append("CREATE SCHEMA IF NOT EXISTS ").append(quote(schema)).append(";\n");
+ try {
+ List<Table> tables = glueService.getTables(schema);
+ appendViews(sb, schema, tables, true);
+ if (schema.equals(contextDatabase)) {
+ appendViews(sb, schema, tables, false);
+ contextDbHandled = true;
+ }
+ } catch (Exception e) {
+ LOG.debugv("Could not fetch tables for Glue database {0}: {1}", schema, e.getMessage());
+ }
+ }
+ }
+
+ if (!contextDbHandled && contextDatabase != null && !contextDatabase.isBlank()) {
+ try {
+ List<Table> tables = glueService.getTables(contextDatabase);
+ appendViews(sb, contextDatabase, tables, false);
+ } catch (Exception e) {
+ LOG.debugv("Could not fetch tables for context database {0}: {1}", contextDatabase, e.getMessage());
+ }
+ }
+
+ return sb.toString();
+ }
+
+ private void appendViews(StringBuilder sb, String schemaOrNull, List<Table> tables, boolean qualified) {
+ if (tables == null) {
+ return;
+ }
+ for (Table t : tables) {
+ try {
+ if (t == null) {
+ continue;
+ }
+ if (t.getName() == null || t.getName().isBlank()) {
+ continue;
+ }
+ if (t.getStorageDescriptor() == null
+ || t.getStorageDescriptor().getLocation() == null
+ || t.getStorageDescriptor().getLocation().isBlank()) {
+ continue;
+ }
+ String location = t.getStorageDescriptor().getLocation();
+ String normalizedLocation = location.endsWith("/")
+ ? location.substring(0, location.length() - 1)
+ : location;
+ String readFn = inferReadFunction(t);
+ String target = qualified
+ ? quote(schemaOrNull) + "." + quote(t.getName())
+ : quote(t.getName());
+ sb.append("CREATE OR REPLACE VIEW ")
+ .append(target)
+ .append(" AS SELECT * FROM ")
+ .append(readExpression(readFn, normalizedLocation))
+ .append(";\n");
+ } catch (Exception e) {
+ LOG.debugv("skip Glue table {0}.{1}: {2}", schemaOrNull, t != null ? t.getName() : "unknown", e.getMessage());
+ }
+ }
+ }
+
+ static String inferReadFunction(Table table) {
+ if (table == null || table.getStorageDescriptor() == null) {
+ return "read_csv_auto";
+ }
+ String format = table.getStorageDescriptor().getInputFormat();
+ String serde = table.getStorageDescriptor().getSerdeInfo() != null
+ ? table.getStorageDescriptor().getSerdeInfo().getSerializationLibrary()
+ : null;
+ if (containsIgnoreCase(format, "parquet") || containsIgnoreCase(serde, "parquet")) {
+ return "read_parquet";
+ }
+ if (containsIgnoreCase(format, "json") || containsIgnoreCase(serde, "json")
+ || containsIgnoreCase(format, "hive")) {
+ return "read_json_auto";
+ }
+ return "read_csv_auto";
+ }
+
+ static String readExpression(String readFn, String normalizedLocation) {
+ String escapedLocation = normalizedLocation.replace("'", "''");
+ String glob = escapedLocation + "/**";
+ if ("read_parquet".equals(readFn)) {
+ return "read_parquet('" + glob + "', union_by_name = true)";
+ }
+ return readFn + "('" + glob + "')";
+ }
+
+ static boolean containsIgnoreCase(String str, String sub) {
+ return str != null && str.toLowerCase(Locale.ROOT).contains(sub);
+ }
+
+ static String quote(String id) {
+ return "\"" + id.replace("\"", "\"\"") + "\"";
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.java b/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.java
new file mode 100644
index 0000000000..4e65ad153e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.java
@@ -0,0 +1,242 @@
+package io.github.hectorvent.floci.services.athena;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.glue.GlueService;
+import io.github.hectorvent.floci.services.glue.model.Database;
+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
+import io.github.hectorvent.floci.services.glue.model.Table;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.when;
+
+class GlueViewDdlBuilderTest {
+
+ private GlueService glueService;
+ private GlueViewDdlBuilder builder;
+
+ @BeforeEach
+ void setUp() {
+ glueService = Mockito.mock(GlueService.class);
+ builder = new GlueViewDdlBuilder(glueService);
+ }
+
+ private Database createDatabase(String name) {
+ Database db = new Database();
+ db.setName(name);
+ return db;
+ }
+
+ private Table createTable(String name, String location, String serde, String inputFormat) {
+ Table table = new Table();
+ table.setName(name);
+ if (location != null || serde != null || inputFormat != null) {
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setLocation(location);
+ sd.setInputFormat(inputFormat);
+ if (serde != null) {
+ StorageDescriptor.SerDeInfo serdeInfo = new StorageDescriptor.SerDeInfo();
+ serdeInfo.setSerializationLibrary(serde);
+ sd.setSerdeInfo(serdeInfo);
+ }
+ table.setStorageDescriptor(sd);
+ }
+ return table;
+ }
+
+ @Test
+ void testTwoDatabasesQualifiedViewsAndSchemaPerDb() {
+ Database shopDb = createDatabase("shop");
+ Database analyticsDb = createDatabase("analytics");
+ when(glueService.getDatabases()).thenReturn(List.of(shopDb, analyticsDb));
+
+ Table ordersTable = createTable("orders", "s3://analytics-bucket/orders/", "org.openx.data.jsonserde.JsonSerDe", null);
+ Table eventsTable = createTable("events", "s3://analytics-bucket/events/", "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", null);
+ when(glueService.getTables("shop")).thenReturn(List.of(ordersTable));
+ when(glueService.getTables("analytics")).thenReturn(List.of(eventsTable));
+
+ String ddl = builder.build(null);
+
+ assertTrue(ddl.contains("CREATE SCHEMA IF NOT EXISTS \"shop\";\n"));
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"shop\".\"orders\" AS SELECT * FROM read_json_auto('s3://analytics-bucket/orders/**');\n"));
+ assertTrue(ddl.contains("CREATE SCHEMA IF NOT EXISTS \"analytics\";\n"));
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"analytics\".\"events\" AS SELECT * FROM read_parquet('s3://analytics-bucket/events/**', union_by_name = true);\n"));
+ assertFalse(ddl.contains("CREATE OR REPLACE VIEW \"orders\" AS"));
+ assertFalse(ddl.contains("CREATE OR REPLACE VIEW \"events\" AS"));
+ }
+
+ @Test
+ void testContextDatabaseAddsUnqualifiedAlias() {
+ Database shopDb = createDatabase("shop");
+ when(glueService.getDatabases()).thenReturn(List.of(shopDb));
+
+ Table ordersTable = createTable("orders", "s3://analytics-bucket/orders", "org.openx.data.jsonserde.JsonSerDe", null);
+ when(glueService.getTables("shop")).thenReturn(List.of(ordersTable));
+
+ String ddlWithContext = builder.build("shop");
+ assertTrue(ddlWithContext.contains("CREATE OR REPLACE VIEW \"shop\".\"orders\" AS SELECT * FROM read_json_auto('s3://analytics-bucket/orders/**');\n"));
+ assertTrue(ddlWithContext.contains("CREATE OR REPLACE VIEW \"orders\" AS SELECT * FROM read_json_auto('s3://analytics-bucket/orders/**');\n"));
+
+ String ddlNull = builder.build(null);
+ assertTrue(ddlNull.contains("CREATE OR REPLACE VIEW \"shop\".\"orders\" AS SELECT * FROM read_json_auto('s3://analytics-bucket/orders/**');\n"));
+ assertFalse(ddlNull.contains("CREATE OR REPLACE VIEW \"orders\" AS"));
+
+ String ddlEmpty = builder.build("");
+ assertTrue(ddlEmpty.contains("CREATE OR REPLACE VIEW \"shop\".\"orders\" AS SELECT * FROM read_json_auto('s3://analytics-bucket/orders/**');\n"));
+ assertFalse(ddlEmpty.contains("CREATE OR REPLACE VIEW \"orders\" AS"));
+ }
+
+ @Test
+ void testContextDatabaseNotInGlueDoesNotThrow() {
+ Database shopDb = createDatabase("shop");
+ when(glueService.getDatabases()).thenReturn(List.of(shopDb));
+
+ Table ordersTable = createTable("orders", "s3://analytics-bucket/orders", "org.openx.data.jsonserde.JsonSerDe", null);
+ when(glueService.getTables("shop")).thenReturn(List.of(ordersTable));
+ when(glueService.getTables("ghost")).thenThrow(new AwsException("EntityNotFoundException", "Database ghost not found", 400));
+
+ String ddl = builder.build("ghost");
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"shop\".\"orders\" AS SELECT * FROM read_json_auto('s3://analytics-bucket/orders/**');\n"));
+ assertFalse(ddl.contains("CREATE OR REPLACE VIEW \"ghost\""));
+ assertFalse(ddl.contains("CREATE OR REPLACE VIEW \"orders\" AS"));
+ }
+
+ @Test
+ void testTableWithNullOrBlankLocationIsSkipped() {
+ Database shopDb = createDatabase("shop");
+ when(glueService.getDatabases()).thenReturn(List.of(shopDb));
+
+ Table noSd = new Table();
+ noSd.setName("no_sd");
+
+ Table nullLoc = createTable("null_loc", null, null, null);
+ Table blankLoc = createTable("blank_loc", " ", null, null);
+ Table valid = createTable("valid", "s3://bucket/valid", null, null);
+
+ when(glueService.getTables("shop")).thenReturn(List.of(noSd, nullLoc, blankLoc, valid));
+
+ String ddl = builder.build(null);
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"shop\".\"valid\" AS SELECT * FROM read_csv_auto('s3://bucket/valid/**');\n"));
+ assertFalse(ddl.contains("no_sd"));
+ assertFalse(ddl.contains("null_loc"));
+ assertFalse(ddl.contains("blank_loc"));
+ }
+
+ @Test
+ void testReadFunctionInferencePreserved() {
+ Database db = createDatabase("db");
+ when(glueService.getDatabases()).thenReturn(List.of(db));
+
+ Table parquetTable = createTable("parquet_tbl", "s3://bucket/parquet", "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", null);
+ Table jsonTable = createTable("json_tbl", "s3://bucket/json", "org.openx.data.jsonserde.JsonSerDe", "org.apache.hadoop.mapred.TextInputFormat");
+ Table hiveTable = createTable("hive_tbl", "s3://bucket/hive", null, "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
+ Table csvTable = createTable("csv_tbl", "s3://bucket/csv", null, null);
+
+ when(glueService.getTables("db")).thenReturn(List.of(parquetTable, jsonTable, hiveTable, csvTable));
+
+ String ddl = builder.build(null);
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"db\".\"parquet_tbl\" AS SELECT * FROM read_parquet('s3://bucket/parquet/**', union_by_name = true);\n"));
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"db\".\"json_tbl\" AS SELECT * FROM read_json_auto('s3://bucket/json/**');\n"));
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"db\".\"hive_tbl\" AS SELECT * FROM read_json_auto('s3://bucket/hive/**');\n"));
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"db\".\"csv_tbl\" AS SELECT * FROM read_csv_auto('s3://bucket/csv/**');\n"));
+ }
+
+ @Test
+ void testIdentifierQuotingEscapesDoubleQuotes() {
+ Database db = createDatabase("my\"db");
+ when(glueService.getDatabases()).thenReturn(List.of(db));
+
+ Table tbl = createTable("my\"table", "s3://bucket/quoted", null, null);
+ when(glueService.getTables("my\"db")).thenReturn(List.of(tbl));
+
+ String ddl = builder.build("my\"db");
+ assertTrue(ddl.contains("CREATE SCHEMA IF NOT EXISTS \"my\"\"db\";\n"));
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"my\"\"db\".\"my\"\"table\" AS SELECT * FROM read_csv_auto('s3://bucket/quoted/**');\n"));
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"my\"\"table\" AS SELECT * FROM read_csv_auto('s3://bucket/quoted/**');\n"));
+ }
+
+ @Test
+ void testNoDatabasesReturnsEmpty() {
+ when(glueService.getDatabases()).thenReturn(List.of());
+ when(glueService.getTables("x")).thenReturn(List.of());
+
+ String ddl = builder.build("x");
+ assertEquals("", ddl);
+ }
+
+ @Test
+ void testNullOrBlankDatabaseNameIsSkipped() {
+ Database nullNameDb = createDatabase(null);
+ Database blankNameDb = createDatabase(" ");
+ Database validDb = createDatabase("valid_db");
+ when(glueService.getDatabases()).thenReturn(java.util.Arrays.asList(null, nullNameDb, blankNameDb, validDb));
+ when(glueService.getTables("valid_db")).thenReturn(List.of());
+
+ String ddl = builder.build(null);
+ assertTrue(ddl.contains("CREATE SCHEMA IF NOT EXISTS \"valid_db\";\n"));
+ assertFalse(ddl.contains("CREATE SCHEMA IF NOT EXISTS \"\";"));
+ assertFalse(ddl.contains("CREATE SCHEMA IF NOT EXISTS \"null\";"));
+ }
+
+ @Test
+ void testGetDatabasesExceptionPropagates() {
+ when(glueService.getDatabases()).thenThrow(new RuntimeException("Glue failure"));
+ org.junit.jupiter.api.Assertions.assertThrows(RuntimeException.class, () -> builder.build(null));
+ }
+
+ @Test
+ void testTableWithNullOrBlankNameIsSkipped() {
+ Database shopDb = createDatabase("shop");
+ when(glueService.getDatabases()).thenReturn(List.of(shopDb));
+
+ Table nullNameTable = createTable(null, "s3://bucket/null_name", null, null);
+ Table emptyNameTable = createTable("", "s3://bucket/empty_name", null, null);
+ Table blankNameTable = createTable(" ", "s3://bucket/blank_name", null, null);
+ Table validTable = createTable("valid_tbl", "s3://bucket/valid", null, null);
+
+ when(glueService.getTables("shop")).thenReturn(List.of(nullNameTable, emptyNameTable, blankNameTable, validTable));
+
+ String ddl = builder.build("shop");
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"shop\".\"valid_tbl\" AS SELECT * FROM read_csv_auto('s3://bucket/valid/**');\n"));
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"valid_tbl\" AS SELECT * FROM read_csv_auto('s3://bucket/valid/**');\n"));
+ assertFalse(ddl.contains("null_name"));
+ assertFalse(ddl.contains("empty_name"));
+ assertFalse(ddl.contains("blank_name"));
+ assertFalse(ddl.contains("CREATE OR REPLACE VIEW \"shop\".\"\""));
+ assertFalse(ddl.contains("CREATE OR REPLACE VIEW \"\""));
+ }
+
+ @Test
+ void testSingleQuotesInLocationAreEscaped() {
+ Database db = createDatabase("db");
+ when(glueService.getDatabases()).thenReturn(List.of(db));
+
+ Table singleQuoteTable = createTable("user_events", "s3://bucket/it's-a-dir/data", null, null);
+ Table parquetSingleQuote = createTable("parquet_events", "s3://bucket/user's-store/parquet", "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", null);
+ when(glueService.getTables("db")).thenReturn(List.of(singleQuoteTable, parquetSingleQuote));
+
+ String ddl = builder.build(null);
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"db\".\"user_events\" AS SELECT * FROM read_csv_auto('s3://bucket/it''s-a-dir/data/**');\n"));
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"db\".\"parquet_events\" AS SELECT * FROM read_parquet('s3://bucket/user''s-store/parquet/**', union_by_name = true);\n"));
+ }
+
+ @Test
+ void testContextDatabaseTablesFetchedOnlyOnce() {
+ Database db = createDatabase("shop");
+ when(glueService.getDatabases()).thenReturn(List.of(db));
+ Table table = createTable("orders", "s3://bucket/data", null, null);
+ when(glueService.getTables("shop")).thenReturn(List.of(table));
+
+ String ddl = builder.build("shop");
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"shop\".\"orders\""));
+ assertTrue(ddl.contains("CREATE OR REPLACE VIEW \"orders\""));
+ Mockito.verify(glueService, Mockito.times(1)).getTables("shop");
+ }
+}
+