Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(athena): route database DDL to Glue (#2757)

Aniket Shukla2480daaa099b
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java
index 52880affaf..0e52929408 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java
@@ -169,4 +169,40 @@ class AthenaTest {
glue.close();
}
+
+ @Test
+ @Order(8)
+ @DisplayName("CREATE DATABASE through Athena updates the Glue catalog")
+ void createDatabaseDdlUsesGlueCatalog() throws InterruptedException {
+ String dbName = "athena_ddl_test_" + UUID.randomUUID().toString().replace("-", "").substring(0, 8);
+ String location = "s3://test-bucket/" + dbName + "/";
+
+ try (GlueClient glue = TestFixtures.glueClient()) {
+ StartQueryExecutionResponse started = athena.startQueryExecution(
+ StartQueryExecutionRequest.builder()
+ .queryString("CREATE DATABASE IF NOT EXISTS " + dbName
+ + " LOCATION '" + location + "'")
+ .workGroup("primary")
+ .build());
+
+ QueryExecutionStatus status = TestFixtures.awaitAthenaQueryTerminal(
+ athena, started.queryExecutionId(), Duration.ofSeconds(60));
+ assertThat(status.state())
+ .as("Athena DDL did not succeed: %s", status.stateChangeReason())
+ .isEqualTo(QueryExecutionState.SUCCEEDED);
+
+ QueryExecution execution = athena.getQueryExecution(r -> r
+ .queryExecutionId(started.queryExecutionId())).queryExecution();
+ assertThat(execution.statementType()).isEqualTo(StatementType.DDL);
+ assertThat(execution.resultConfiguration()).isNull();
+
+ GetQueryResultsResponse results = athena.getQueryResults(r -> r
+ .queryExecutionId(started.queryExecutionId()));
+ assertThat(results.resultSet().rows()).isEmpty();
+ assertThat(results.resultSet().resultSetMetadata().columnInfo()).isEmpty();
+
+ assertThat(glue.getDatabase(r -> r.name(dbName)).database().locationUri())
+ .isEqualTo(location);
+ }
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
index 3f075f3006..0122d00073 100644
--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
@@ -25,6 +25,8 @@ import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
@ApplicationScoped
public class AthenaService {
@@ -34,6 +36,23 @@ public class AthenaService {
private static final String DEFAULT_OUTPUT_BUCKET = "floci-athena-results";
private static final String DEFAULT_WORKGROUP = "primary";
private static final String DEFAULT_ENGINE_VERSION = "Athena engine version 3";
+ private static final Pattern CREATE_DATABASE_PATTERN = Pattern.compile(
+ "^\\s*CREATE\\s+(?:DATABASE|SCHEMA)\\s+"
+ + "(IF\\s+NOT\\s+EXISTS\\s+)?"
+ + "(?:`([^`]+)`|\\\"([^\\\"]+)\\\"|([a-zA-Z0-9_-]+))"
+ + "(?:\\s+COMMENT\\s+'((?:''|[^'])*)')?"
+ + "(?:\\s+LOCATION\\s+'((?:''|[^'])*)')?"
+ + "(?:\\s+WITH\\s+DBPROPERTIES\\s*\\((.*?)\\))?"
+ + "\\s*;?\\s*$",
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+ private static final Pattern CREATE_DATABASE_PREFIX_PATTERN = Pattern.compile(
+ "^\\s*CREATE\\s+(?:DATABASE|SCHEMA)\\b",
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+ private static final Pattern DATABASE_PROPERTY_PATTERN = Pattern.compile(
+ "\\s*'((?:''|[^'])*)'\\s*=\\s*'((?:''|[^'])*)'\\s*");
+ private static final Pattern RESULT_STATEMENT_PATTERN = Pattern.compile(
+ "^(?:SELECT|WITH|SHOW|DESCRIBE|DESC|EXPLAIN|VALUES)\\b",
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
private final StorageBackend<String, QueryExecution> queryStore;
private final StorageBackend<String, WorkGroup> workGroupStore;
@@ -73,19 +92,40 @@ public class AthenaService {
resolvedContext.setCatalog(DEFAULT_CATALOG);
}
+ boolean createDatabaseStatement = CREATE_DATABASE_PREFIX_PATTERN.matcher(statementText(query)).find();
// AWS reports the result CSV object itself as the OutputLocation, not a
// directory prefix — the same key is written and returned to the client.
- String outputLocation = resolveOutputLocation(resultConfiguration, id);
- ResultConfiguration resolvedResult = new ResultConfiguration(outputLocation);
+ // Statements that do not return rows must not be wrapped in a result COPY.
+ String outputLocation = producesResultRows(query)
+ ? resolveOutputLocation(resultConfiguration, id)
+ : null;
+ ResultConfiguration resolvedResult = outputLocation != null
+ ? new ResultConfiguration(outputLocation)
+ : null;
QueryExecution execution = new QueryExecution(id, query, workGroup, resolvedResult, resolvedContext);
+ if (createDatabaseStatement) {
+ execution.setStatementType("DDL");
+ }
execution.getStatus().setState(QueryExecutionState.RUNNING);
queryStore.put(id, execution);
+ if (createDatabaseStatement) {
+ try {
+ CreateDatabaseDdl createDatabaseDdl = parseCreateDatabase(query);
+ if (createDatabaseDdl == null) {
+ throw new AwsException("InvalidRequestException", "Invalid CREATE DATABASE statement", 400);
+ }
+ createGlueDatabase(createDatabaseDdl);
+ markSucceeded(id, execution);
+ } catch (Exception e) {
+ markFailed(id, execution, e);
+ }
+ return id;
+ }
+
if (config.services().athena().mock()) {
- execution.getStatus().setState(QueryExecutionState.SUCCEEDED);
- execution.getStatus().setCompletionDateTime(Instant.now());
- queryStore.put(id, execution);
+ markSucceeded(id, execution);
LOG.infov("Query {0} accepted (mock mode)", id);
return id;
}
@@ -93,18 +133,16 @@ public class AthenaService {
// Submit async — caller gets the ID immediately while execution runs in background
vertx.executeBlocking(() -> {
String setupDdl = buildGlueDdl(database);
- ensureOutputBucket(outputLocation);
+ if (outputLocation != null) {
+ ensureOutputBucket(outputLocation);
+ }
duckClient.execute(query, setupDdl, outputLocation);
return null;
}).onSuccess(v -> {
- execution.getStatus().setState(QueryExecutionState.SUCCEEDED);
- execution.getStatus().setCompletionDateTime(Instant.now());
- queryStore.put(id, execution);
+ markSucceeded(id, execution);
LOG.infov("Query {0} succeeded", id);
}).onFailure(e -> {
- execution.getStatus().setState(QueryExecutionState.FAILED);
- execution.getStatus().setStateChangeReason(e.getMessage());
- queryStore.put(id, execution);
+ markFailed(id, execution, e);
LOG.warnv("Query {0} failed: {1}", id, e.getMessage());
});
@@ -220,6 +258,115 @@ public class AthenaService {
// ── private helpers ───────────────────────────────────────────────────────
+ private CreateDatabaseDdl parseCreateDatabase(String query) {
+ Matcher matcher = CREATE_DATABASE_PATTERN.matcher(statementText(query));
+ if (!matcher.matches()) {
+ return null;
+ }
+ String name = firstNonNull(matcher.group(2), matcher.group(3), matcher.group(4));
+ return new CreateDatabaseDdl(
+ name,
+ matcher.group(1) != null,
+ unescapeSqlString(matcher.group(5)),
+ unescapeSqlString(matcher.group(6)),
+ parseDatabaseProperties(matcher.group(7)));
+ }
+
+ private void createGlueDatabase(CreateDatabaseDdl statement) {
+ Database database = new Database(statement.name());
+ database.setDescription(statement.comment());
+ database.setLocationUri(statement.location());
+ database.setParameters(statement.properties());
+ try {
+ glueService.createDatabase(database);
+ } catch (AwsException e) {
+ if (!statement.ifNotExists() || !"AlreadyExistsException".equals(e.getErrorCode())) {
+ throw e;
+ }
+ }
+ }
+
+ private static Map<String, String> parseDatabaseProperties(String clause) {
+ if (clause == null) {
+ return null;
+ }
+ Map<String, String> properties = new LinkedHashMap<>();
+ int cursor = 0;
+ while (cursor < clause.length()) {
+ Matcher matcher = DATABASE_PROPERTY_PATTERN.matcher(clause);
+ matcher.region(cursor, clause.length());
+ if (!matcher.lookingAt()) {
+ throw new AwsException("InvalidRequestException", "Invalid database properties", 400);
+ }
+ properties.put(unescapeSqlString(matcher.group(1)), unescapeSqlString(matcher.group(2)));
+ cursor = matcher.end();
+ if (cursor == clause.length()) {
+ break;
+ }
+ if (clause.charAt(cursor) != ',') {
+ throw new AwsException("InvalidRequestException", "Invalid database properties", 400);
+ }
+ cursor++;
+ if (clause.substring(cursor).isBlank()) {
+ throw new AwsException("InvalidRequestException", "Invalid database properties", 400);
+ }
+ }
+ return properties;
+ }
+
+ private static boolean producesResultRows(String query) {
+ return RESULT_STATEMENT_PATTERN.matcher(statementText(query)).find();
+ }
+
+ private static String statementText(String query) {
+ String statement = query == null ? "" : query.stripLeading();
+ while (true) {
+ if (statement.startsWith("--")) {
+ int lineEnd = statement.indexOf('\n');
+ statement = lineEnd >= 0 ? statement.substring(lineEnd + 1).stripLeading() : "";
+ continue;
+ }
+ if (statement.startsWith("/*")) {
+ int commentEnd = statement.indexOf("*/", 2);
+ if (commentEnd < 0) {
+ return statement;
+ }
+ statement = statement.substring(commentEnd + 2).stripLeading();
+ continue;
+ }
+ return statement;
+ }
+ }
+
+ private static String firstNonNull(String... values) {
+ for (String value : values) {
+ if (value != null) {
+ return value;
+ }
+ }
+ return null;
+ }
+
+ private static String unescapeSqlString(String value) {
+ return value == null ? null : value.replace("''", "'");
+ }
+
+ private void markSucceeded(String id, QueryExecution execution) {
+ execution.getStatus().setState(QueryExecutionState.SUCCEEDED);
+ execution.getStatus().setCompletionDateTime(Instant.now());
+ queryStore.put(id, execution);
+ }
+
+ private void markFailed(String id, QueryExecution execution, Throwable failure) {
+ execution.getStatus().setState(QueryExecutionState.FAILED);
+ execution.getStatus().setStateChangeReason(failure.getMessage());
+ queryStore.put(id, execution);
+ }
+
+ private record CreateDatabaseDdl(String name, boolean ifNotExists, String comment,
+ String location, Map<String, String> properties) {
+ }
+
private String buildGlueDdl(String database) {
StringBuilder sb = new StringBuilder();
try {
diff --git a/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClient.java b/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClient.java
index 534a3d78d2..5f6b5be6a1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClient.java
+++ b/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClient.java
@@ -61,7 +61,8 @@ public class FlociDuckClient {
*
* @param sql the SQL to run (FROM clauses reference s3:// paths)
* @param setupDdl optional DDL to run before the query (e.g. CREATE VIEW)
- * @param outputS3Path the S3 path where the result CSV will be written
+ * @param outputS3Path the S3 path where the result CSV will be written,
+ * or {@code null} to execute without exporting results
*/
public void execute(String sql, String setupDdl, String outputS3Path) {
execute(sql, setupDdl, outputS3Path, null);
@@ -79,7 +80,9 @@ public class FlociDuckClient {
String duckUrl = duckManager.ensureReady();
Map<String, Object> body = buildBaseBody(sql, setupDdl, accessKeyId);
- body.put("output_s3_path", outputS3Path);
+ if (outputS3Path != null) {
+ body.put("output_s3_path", outputS3Path);
+ }
post(duckUrl + "/execute", body, "execute");
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.java
index a83ea6bbf1..ce70355874 100644
--- a/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.java
@@ -4,6 +4,8 @@ import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
import org.junit.jupiter.api.*;
+import java.util.Map;
+
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.*;
@@ -408,4 +410,110 @@ class AthenaIntegrationTest {
.body("QueryExecution.ResultConfiguration.OutputLocation",
equalTo("s3://athena-location-test/out/" + id + ".csv"));
}
+
+ @Test
+ @Order(14)
+ void createDatabaseDdlUpdatesGlueCatalogWithoutResultOutput() {
+ String query = """
+ CREATE DATABASE IF NOT EXISTS workspace_dev_test
+ COMMENT 'Created through Athena'
+ LOCATION 's3://bucket/path'
+ WITH DBPROPERTIES ('owner'='integration-test')
+ """;
+ String id = given()
+ .header("X-Amz-Target", "AmazonAthena.StartQueryExecution")
+ .contentType(CONTENT_TYPE)
+ .body(Map.of("QueryString", query))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().path("QueryExecutionId");
+
+ given()
+ .header("X-Amz-Target", "AmazonAthena.GetQueryExecution")
+ .contentType(CONTENT_TYPE)
+ .body("{ \"QueryExecutionId\": \"" + id + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("QueryExecution.Status.State", equalTo("SUCCEEDED"))
+ .body("QueryExecution.StatementType", equalTo("DDL"))
+ .body("QueryExecution.ResultConfiguration", nullValue());
+
+ given()
+ .header("X-Amz-Target", "AWSGlue.GetDatabase")
+ .contentType(CONTENT_TYPE)
+ .body("{ \"Name\": \"workspace_dev_test\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("Database.Name", equalTo("workspace_dev_test"))
+ .body("Database.Description", equalTo("Created through Athena"))
+ .body("Database.LocationUri", equalTo("s3://bucket/path"))
+ .body("Database.Parameters.owner", equalTo("integration-test"));
+
+ given()
+ .header("X-Amz-Target", "AmazonAthena.GetQueryResults")
+ .contentType(CONTENT_TYPE)
+ .body("{ \"QueryExecutionId\": \"" + id + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("ResultSet.Rows", empty())
+ .body("ResultSet.ResultSetMetadata.ColumnInfo", empty());
+
+ String repeatId = given()
+ .header("X-Amz-Target", "AmazonAthena.StartQueryExecution")
+ .contentType(CONTENT_TYPE)
+ .body(Map.of("QueryString", query))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().path("QueryExecutionId");
+
+ given()
+ .header("X-Amz-Target", "AmazonAthena.GetQueryExecution")
+ .contentType(CONTENT_TYPE)
+ .body("{ \"QueryExecutionId\": \"" + repeatId + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("QueryExecution.Status.State", equalTo("SUCCEEDED"));
+ }
+
+ @Test
+ @Order(15)
+ void malformedCreateDatabaseDdlIsRecordedAsFailedExecution() {
+ String id = given()
+ .header("X-Amz-Target", "AmazonAthena.StartQueryExecution")
+ .contentType(CONTENT_TYPE)
+ .body(Map.of("QueryString", """
+ CREATE DATABASE invalid_properties_test
+ WITH DBPROPERTIES ('owner'='integration-test', )
+ """))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().path("QueryExecutionId");
+
+ given()
+ .header("X-Amz-Target", "AmazonAthena.GetQueryExecution")
+ .contentType(CONTENT_TYPE)
+ .body("{ \"QueryExecutionId\": \"" + id + "\" }")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("QueryExecution.Status.State", equalTo("FAILED"))
+ .body("QueryExecution.Status.StateChangeReason", equalTo("Invalid database properties"))
+ .body("QueryExecution.StatementType", equalTo("DDL"))
+ .body("QueryExecution.ResultConfiguration", nullValue());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClientServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClientServiceTest.java
new file mode 100644
index 0000000000..83240f1f75
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClientServiceTest.java
@@ -0,0 +1,63 @@
+package io.github.hectorvent.floci.services.floci.duck;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.sun.net.httpserver.HttpServer;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+class FlociDuckClientServiceTest {
+
+ private HttpServer server;
+
+ @AfterEach
+ void stopServer() {
+ if (server != null) {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void executeOmitsNullOutputPath() throws Exception {
+ ObjectMapper mapper = new ObjectMapper();
+ AtomicReference<JsonNode> requestBody = new AtomicReference<>();
+ server = HttpServer.create(new InetSocketAddress(0), 0);
+ server.createContext("/execute", exchange -> {
+ String body = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8);
+ requestBody.set(mapper.readTree(body));
+ exchange.sendResponseHeaders(200, -1);
+ exchange.close();
+ });
+ server.start();
+
+ FlociDuckManager duckManager = mock(FlociDuckManager.class);
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ EmbeddedDnsServer embeddedDnsServer = mock(EmbeddedDnsServer.class);
+ DockerHostResolver dockerHostResolver = mock(DockerHostResolver.class);
+ when(duckManager.ensureReady()).thenReturn("http://localhost:" + server.getAddress().getPort());
+ when(config.baseUrl()).thenReturn("http://localhost:4566");
+ when(config.defaultRegion()).thenReturn("us-east-1");
+ when(config.hostname()).thenReturn(Optional.of("floci.internal"));
+ when(embeddedDnsServer.getServerIp()).thenReturn(Optional.of("127.0.0.1"));
+
+ FlociDuckClient client = new FlociDuckClient(
+ duckManager, config, embeddedDnsServer, dockerHostResolver, mapper);
+ client.execute("CREATE VIEW example AS SELECT 1", "", null);
+
+ assertEquals("CREATE VIEW example AS SELECT 1", requestBody.get().path("sql").asText());
+ assertFalse(requestBody.get().has("output_s3_path"));
+ }
+}