Light, fluffy, and always free - The AWS Local Emulator alternative
fix(athena): route database DDL to Glue (#2757)
Aniket Shukla2480daaa099b
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.javaindex 52880affaf..0e52929408 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java@@ -169,4 +169,40 @@ class AthenaTest {glue.close();}++ @Test+ @Order(8)+ @DisplayName("CREATE DATABASE through Athena updates the Glue catalog")+ void createDatabaseDdlUsesGlueCatalog() throws InterruptedException {+ String dbName = "athena_ddl_test_" + UUID.randomUUID().toString().replace("-", "").substring(0, 8);+ String location = "s3://test-bucket/" + dbName + "/";++ try (GlueClient glue = TestFixtures.glueClient()) {+ StartQueryExecutionResponse started = athena.startQueryExecution(+ StartQueryExecutionRequest.builder()+ .queryString("CREATE DATABASE IF NOT EXISTS " + dbName+ + " LOCATION '" + location + "'")+ .workGroup("primary")+ .build());++ QueryExecutionStatus status = TestFixtures.awaitAthenaQueryTerminal(+ athena, started.queryExecutionId(), Duration.ofSeconds(60));+ assertThat(status.state())+ .as("Athena DDL did not succeed: %s", status.stateChangeReason())+ .isEqualTo(QueryExecutionState.SUCCEEDED);++ QueryExecution execution = athena.getQueryExecution(r -> r+ .queryExecutionId(started.queryExecutionId())).queryExecution();+ assertThat(execution.statementType()).isEqualTo(StatementType.DDL);+ assertThat(execution.resultConfiguration()).isNull();++ GetQueryResultsResponse results = athena.getQueryResults(r -> r+ .queryExecutionId(started.queryExecutionId()));+ assertThat(results.resultSet().rows()).isEmpty();+ assertThat(results.resultSet().resultSetMetadata().columnInfo()).isEmpty();++ assertThat(glue.getDatabase(r -> r.name(dbName)).database().locationUri())+ .isEqualTo(location);+ }+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.javaindex 3f075f3006..0122d00073 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java@@ -25,6 +25,8 @@ import java.io.InputStreamReader;import java.nio.charset.StandardCharsets;import java.time.Instant;import java.util.*;+import java.util.regex.Matcher;+import java.util.regex.Pattern;@ApplicationScopedpublic class AthenaService {@@ -34,6 +36,23 @@ public class AthenaService {private static final String DEFAULT_OUTPUT_BUCKET = "floci-athena-results";private static final String DEFAULT_WORKGROUP = "primary";private static final String DEFAULT_ENGINE_VERSION = "Athena engine version 3";+ private static final Pattern CREATE_DATABASE_PATTERN = Pattern.compile(+ "^\\s*CREATE\\s+(?:DATABASE|SCHEMA)\\s+"+ + "(IF\\s+NOT\\s+EXISTS\\s+)?"+ + "(?:`([^`]+)`|\\\"([^\\\"]+)\\\"|([a-zA-Z0-9_-]+))"+ + "(?:\\s+COMMENT\\s+'((?:''|[^'])*)')?"+ + "(?:\\s+LOCATION\\s+'((?:''|[^'])*)')?"+ + "(?:\\s+WITH\\s+DBPROPERTIES\\s*\\((.*?)\\))?"+ + "\\s*;?\\s*$",+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL);+ private static final Pattern CREATE_DATABASE_PREFIX_PATTERN = Pattern.compile(+ "^\\s*CREATE\\s+(?:DATABASE|SCHEMA)\\b",+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL);+ private static final Pattern DATABASE_PROPERTY_PATTERN = Pattern.compile(+ "\\s*'((?:''|[^'])*)'\\s*=\\s*'((?:''|[^'])*)'\\s*");+ private static final Pattern RESULT_STATEMENT_PATTERN = Pattern.compile(+ "^(?:SELECT|WITH|SHOW|DESCRIBE|DESC|EXPLAIN|VALUES)\\b",+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL);private final StorageBackend<String, QueryExecution> queryStore;private final StorageBackend<String, WorkGroup> workGroupStore;@@ -73,19 +92,40 @@ public class AthenaService {resolvedContext.setCatalog(DEFAULT_CATALOG);}+ boolean createDatabaseStatement = CREATE_DATABASE_PREFIX_PATTERN.matcher(statementText(query)).find();// AWS reports the result CSV object itself as the OutputLocation, not a// directory prefix — the same key is written and returned to the client.- String outputLocation = resolveOutputLocation(resultConfiguration, id);- ResultConfiguration resolvedResult = new ResultConfiguration(outputLocation);+ // Statements that do not return rows must not be wrapped in a result COPY.+ String outputLocation = producesResultRows(query)+ ? resolveOutputLocation(resultConfiguration, id)+ : null;+ ResultConfiguration resolvedResult = outputLocation != null+ ? new ResultConfiguration(outputLocation)+ : null;QueryExecution execution = new QueryExecution(id, query, workGroup, resolvedResult, resolvedContext);+ if (createDatabaseStatement) {+ execution.setStatementType("DDL");+ }execution.getStatus().setState(QueryExecutionState.RUNNING);queryStore.put(id, execution);+ if (createDatabaseStatement) {+ try {+ CreateDatabaseDdl createDatabaseDdl = parseCreateDatabase(query);+ if (createDatabaseDdl == null) {+ throw new AwsException("InvalidRequestException", "Invalid CREATE DATABASE statement", 400);+ }+ createGlueDatabase(createDatabaseDdl);+ markSucceeded(id, execution);+ } catch (Exception e) {+ markFailed(id, execution, e);+ }+ return id;+ }+if (config.services().athena().mock()) {- execution.getStatus().setState(QueryExecutionState.SUCCEEDED);- execution.getStatus().setCompletionDateTime(Instant.now());- queryStore.put(id, execution);+ markSucceeded(id, execution);LOG.infov("Query {0} accepted (mock mode)", id);return id;}@@ -93,18 +133,16 @@ public class AthenaService {// Submit async — caller gets the ID immediately while execution runs in backgroundvertx.executeBlocking(() -> {String setupDdl = buildGlueDdl(database);- ensureOutputBucket(outputLocation);+ if (outputLocation != null) {+ ensureOutputBucket(outputLocation);+ }duckClient.execute(query, setupDdl, outputLocation);return null;}).onSuccess(v -> {- execution.getStatus().setState(QueryExecutionState.SUCCEEDED);- execution.getStatus().setCompletionDateTime(Instant.now());- queryStore.put(id, execution);+ markSucceeded(id, execution);LOG.infov("Query {0} succeeded", id);}).onFailure(e -> {- execution.getStatus().setState(QueryExecutionState.FAILED);- execution.getStatus().setStateChangeReason(e.getMessage());- queryStore.put(id, execution);+ markFailed(id, execution, e);LOG.warnv("Query {0} failed: {1}", id, e.getMessage());});@@ -220,6 +258,115 @@ public class AthenaService {// ── private helpers ───────────────────────────────────────────────────────+ private CreateDatabaseDdl parseCreateDatabase(String query) {+ Matcher matcher = CREATE_DATABASE_PATTERN.matcher(statementText(query));+ if (!matcher.matches()) {+ return null;+ }+ String name = firstNonNull(matcher.group(2), matcher.group(3), matcher.group(4));+ return new CreateDatabaseDdl(+ name,+ matcher.group(1) != null,+ unescapeSqlString(matcher.group(5)),+ unescapeSqlString(matcher.group(6)),+ parseDatabaseProperties(matcher.group(7)));+ }++ private void createGlueDatabase(CreateDatabaseDdl statement) {+ Database database = new Database(statement.name());+ database.setDescription(statement.comment());+ database.setLocationUri(statement.location());+ database.setParameters(statement.properties());+ try {+ glueService.createDatabase(database);+ } catch (AwsException e) {+ if (!statement.ifNotExists() || !"AlreadyExistsException".equals(e.getErrorCode())) {+ throw e;+ }+ }+ }++ private static Map<String, String> parseDatabaseProperties(String clause) {+ if (clause == null) {+ return null;+ }+ Map<String, String> properties = new LinkedHashMap<>();+ int cursor = 0;+ while (cursor < clause.length()) {+ Matcher matcher = DATABASE_PROPERTY_PATTERN.matcher(clause);+ matcher.region(cursor, clause.length());+ if (!matcher.lookingAt()) {+ throw new AwsException("InvalidRequestException", "Invalid database properties", 400);+ }+ properties.put(unescapeSqlString(matcher.group(1)), unescapeSqlString(matcher.group(2)));+ cursor = matcher.end();+ if (cursor == clause.length()) {+ break;+ }+ if (clause.charAt(cursor) != ',') {+ throw new AwsException("InvalidRequestException", "Invalid database properties", 400);+ }+ cursor++;+ if (clause.substring(cursor).isBlank()) {+ throw new AwsException("InvalidRequestException", "Invalid database properties", 400);+ }+ }+ return properties;+ }++ private static boolean producesResultRows(String query) {+ return RESULT_STATEMENT_PATTERN.matcher(statementText(query)).find();+ }++ private static String statementText(String query) {+ String statement = query == null ? "" : query.stripLeading();+ while (true) {+ if (statement.startsWith("--")) {+ int lineEnd = statement.indexOf('\n');+ statement = lineEnd >= 0 ? statement.substring(lineEnd + 1).stripLeading() : "";+ continue;+ }+ if (statement.startsWith("/*")) {+ int commentEnd = statement.indexOf("*/", 2);+ if (commentEnd < 0) {+ return statement;+ }+ statement = statement.substring(commentEnd + 2).stripLeading();+ continue;+ }+ return statement;+ }+ }++ private static String firstNonNull(String... values) {+ for (String value : values) {+ if (value != null) {+ return value;+ }+ }+ return null;+ }++ private static String unescapeSqlString(String value) {+ return value == null ? null : value.replace("''", "'");+ }++ private void markSucceeded(String id, QueryExecution execution) {+ execution.getStatus().setState(QueryExecutionState.SUCCEEDED);+ execution.getStatus().setCompletionDateTime(Instant.now());+ queryStore.put(id, execution);+ }++ private void markFailed(String id, QueryExecution execution, Throwable failure) {+ execution.getStatus().setState(QueryExecutionState.FAILED);+ execution.getStatus().setStateChangeReason(failure.getMessage());+ queryStore.put(id, execution);+ }++ private record CreateDatabaseDdl(String name, boolean ifNotExists, String comment,+ String location, Map<String, String> properties) {+ }+private String buildGlueDdl(String database) {StringBuilder sb = new StringBuilder();try {diff --git a/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClient.java b/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClient.javaindex 534a3d78d2..5f6b5be6a1 100644--- a/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClient.java+++ b/src/main/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClient.java@@ -61,7 +61,8 @@ public class FlociDuckClient {** @param sql the SQL to run (FROM clauses reference s3:// paths)* @param setupDdl optional DDL to run before the query (e.g. CREATE VIEW)- * @param outputS3Path the S3 path where the result CSV will be written+ * @param outputS3Path the S3 path where the result CSV will be written,+ * or {@code null} to execute without exporting results*/public void execute(String sql, String setupDdl, String outputS3Path) {execute(sql, setupDdl, outputS3Path, null);@@ -79,7 +80,9 @@ public class FlociDuckClient {String duckUrl = duckManager.ensureReady();Map<String, Object> body = buildBaseBody(sql, setupDdl, accessKeyId);- body.put("output_s3_path", outputS3Path);+ if (outputS3Path != null) {+ body.put("output_s3_path", outputS3Path);+ }post(duckUrl + "/execute", body, "execute");}diff --git a/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.javaindex a83ea6bbf1..ce70355874 100644--- a/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.java@@ -4,6 +4,8 @@ import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;import org.junit.jupiter.api.*;+import java.util.Map;+import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.*;@@ -408,4 +410,110 @@ class AthenaIntegrationTest {.body("QueryExecution.ResultConfiguration.OutputLocation",equalTo("s3://athena-location-test/out/" + id + ".csv"));}++ @Test+ @Order(14)+ void createDatabaseDdlUpdatesGlueCatalogWithoutResultOutput() {+ String query = """+ CREATE DATABASE IF NOT EXISTS workspace_dev_test+ COMMENT 'Created through Athena'+ LOCATION 's3://bucket/path'+ WITH DBPROPERTIES ('owner'='integration-test')+ """;+ String id = given()+ .header("X-Amz-Target", "AmazonAthena.StartQueryExecution")+ .contentType(CONTENT_TYPE)+ .body(Map.of("QueryString", query))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().path("QueryExecutionId");++ given()+ .header("X-Amz-Target", "AmazonAthena.GetQueryExecution")+ .contentType(CONTENT_TYPE)+ .body("{ \"QueryExecutionId\": \"" + id + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("QueryExecution.Status.State", equalTo("SUCCEEDED"))+ .body("QueryExecution.StatementType", equalTo("DDL"))+ .body("QueryExecution.ResultConfiguration", nullValue());++ given()+ .header("X-Amz-Target", "AWSGlue.GetDatabase")+ .contentType(CONTENT_TYPE)+ .body("{ \"Name\": \"workspace_dev_test\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Database.Name", equalTo("workspace_dev_test"))+ .body("Database.Description", equalTo("Created through Athena"))+ .body("Database.LocationUri", equalTo("s3://bucket/path"))+ .body("Database.Parameters.owner", equalTo("integration-test"));++ given()+ .header("X-Amz-Target", "AmazonAthena.GetQueryResults")+ .contentType(CONTENT_TYPE)+ .body("{ \"QueryExecutionId\": \"" + id + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ResultSet.Rows", empty())+ .body("ResultSet.ResultSetMetadata.ColumnInfo", empty());++ String repeatId = given()+ .header("X-Amz-Target", "AmazonAthena.StartQueryExecution")+ .contentType(CONTENT_TYPE)+ .body(Map.of("QueryString", query))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().path("QueryExecutionId");++ given()+ .header("X-Amz-Target", "AmazonAthena.GetQueryExecution")+ .contentType(CONTENT_TYPE)+ .body("{ \"QueryExecutionId\": \"" + repeatId + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("QueryExecution.Status.State", equalTo("SUCCEEDED"));+ }++ @Test+ @Order(15)+ void malformedCreateDatabaseDdlIsRecordedAsFailedExecution() {+ String id = given()+ .header("X-Amz-Target", "AmazonAthena.StartQueryExecution")+ .contentType(CONTENT_TYPE)+ .body(Map.of("QueryString", """+ CREATE DATABASE invalid_properties_test+ WITH DBPROPERTIES ('owner'='integration-test', )+ """))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().path("QueryExecutionId");++ given()+ .header("X-Amz-Target", "AmazonAthena.GetQueryExecution")+ .contentType(CONTENT_TYPE)+ .body("{ \"QueryExecutionId\": \"" + id + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("QueryExecution.Status.State", equalTo("FAILED"))+ .body("QueryExecution.Status.StateChangeReason", equalTo("Invalid database properties"))+ .body("QueryExecution.StatementType", equalTo("DDL"))+ .body("QueryExecution.ResultConfiguration", nullValue());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClientServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClientServiceTest.javanew file mode 100644index 0000000000..83240f1f75--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/floci/duck/FlociDuckClientServiceTest.java@@ -0,0 +1,63 @@+package io.github.hectorvent.floci.services.floci.duck;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++import java.net.InetSocketAddress;+import java.nio.charset.StandardCharsets;+import java.util.Optional;+import java.util.concurrent.atomic.AtomicReference;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.sun.net.httpserver.HttpServer;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.Test;++class FlociDuckClientServiceTest {++ private HttpServer server;++ @AfterEach+ void stopServer() {+ if (server != null) {+ server.stop(0);+ }+ }++ @Test+ void executeOmitsNullOutputPath() throws Exception {+ ObjectMapper mapper = new ObjectMapper();+ AtomicReference<JsonNode> requestBody = new AtomicReference<>();+ server = HttpServer.create(new InetSocketAddress(0), 0);+ server.createContext("/execute", exchange -> {+ String body = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8);+ requestBody.set(mapper.readTree(body));+ exchange.sendResponseHeaders(200, -1);+ exchange.close();+ });+ server.start();++ FlociDuckManager duckManager = mock(FlociDuckManager.class);+ EmulatorConfig config = mock(EmulatorConfig.class);+ EmbeddedDnsServer embeddedDnsServer = mock(EmbeddedDnsServer.class);+ DockerHostResolver dockerHostResolver = mock(DockerHostResolver.class);+ when(duckManager.ensureReady()).thenReturn("http://localhost:" + server.getAddress().getPort());+ when(config.baseUrl()).thenReturn("http://localhost:4566");+ when(config.defaultRegion()).thenReturn("us-east-1");+ when(config.hostname()).thenReturn(Optional.of("floci.internal"));+ when(embeddedDnsServer.getServerIp()).thenReturn(Optional.of("127.0.0.1"));++ FlociDuckClient client = new FlociDuckClient(+ duckManager, config, embeddedDnsServer, dockerHostResolver, mapper);+ client.execute("CREATE VIEW example AS SELECT 1", "", null);++ assertEquals("CREATE VIEW example AS SELECT 1", requestBody.get().path("sql").asText());+ assertFalse(requestBody.get().has("output_s3_path"));+ }+}