Light, fluffy, and always free - The AWS Local Emulator alternative
feat(athena): real SQL execution via floci-duck DuckDB sidecar (#584)
Hector Ventura1ddbec0efb41
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f34bea79bb..f11b6164e4 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -3,6 +3,10 @@### Features+* **athena:** real SQL execution via `floci-duck` DuckDB sidecar — queries run against actual S3 data; Glue tables are auto-registered as DuckDB views at query time+* **athena:** lazy sidecar lifecycle — `floci/floci-duck:latest` container is started on first query and reused for subsequent executions; `FLOCI_SERVICES_ATHENA_DUCK_URL` bypasses container management+* **athena:** `mock` mode (`FLOCI_SERVICES_ATHENA_MOCK=true`) keeps the previous behavior for unit tests that only verify the state machine+* **glue,athena:** SerDe-aware format inference — `inferReadFunction` now checks `SerdeInfo.SerializationLibrary` in addition to `InputFormat`; standard AWS JSON tables (`TextInputFormat` + `JsonSerDe`) correctly map to `read_json_auto`* **bedrock-runtime:** add stub for Converse and InvokeModel ([#87](https://github.com/floci-io/floci/issues/87))* **s3:** preserve explicit object server-side-encryption headers on PutObject, GetObject, HeadObject, CopyObject, and multipart uploads* **cognito,kms:** allow caller-pinned resource IDs via the reserved `floci:override-id` tag channel ([#460](https://github.com/floci-io/floci/issues/460))@@ -16,6 +20,7 @@* honor `floci.storage.services.acm.*` overrides in `StorageFactory`* **dynamodb:** serialise concurrent mutations via per-item locks and total-ordered transaction lock acquisition so `UpdateItem` counters, conditional `PutItem`/`DeleteItem`, and `TransactWriteItems` are linearisable under load — callers whose prior races silently succeeded may now see the `ConditionalCheckFailedException` / `TransactionCanceledException` that real DynamoDB would raise ([#571](https://github.com/floci-io/floci/issues/571))* **s3:** honor canned object ACLs on PutObject, CopyObject, and multipart uploads+* **lambda:** skip Docker Hub pull when image already exists locally, preventing local builds from being overwritten## [1.5.2](https://github.com/floci-io/floci/compare/1.5.1...1.5.2) (2026-04-10)diff --git a/README.md b/README.mdindex 0f7a98f9c7..804363e949 100644--- a/README.md+++ b/README.md@@ -57,7 +57,7 @@| ElastiCache (Redis + IAM auth) | ✅ | ❌ || RDS (PostgreSQL + MySQL + IAM auth) | ✅ | ❌ || MSK (Kafka + Redpanda) | ✅ | ❌ |-| Athena (query state machine, mock mode) | ✅ | ❌ |+| Athena (real SQL via DuckDB sidecar + Glue views) | ✅ | ❌ || Glue Data Catalog | ✅ | ❌ || Data Firehose (NDJSON delivery) | ✅ | ❌ || S3 Object Lock (COMPLIANCE / GOVERNANCE) | ✅ | ⚠️ Partial |@@ -92,6 +92,7 @@ flowchart LRsubgraph Containers ["Container Services 🐳"]C["Lambda\nElastiCache\nRDS\nECS\nMSK\nEKS"]+ D["Athena ➜ floci-duck\n(DuckDB sidecar)"]endRouter --> Stateless@@ -196,8 +197,8 @@ All default images are configurable via environment variables, useful for pinnin| **ElastiCache** | **Real Docker containers** | Redis / Valkey, IAM auth, SigV4 validation || **RDS** | **Real Docker containers** | PostgreSQL & MySQL, IAM auth, JDBC-compatible || **MSK** | **Real Docker containers** | Kafka compatible via Redpanda orchestration |-| **Athena** | In-process | Query state machine (mock mode — queries accepted, results empty) |-| **Glue** | In-process | Data Catalog for metadata management |+| **Athena** | In-process + **DuckDB sidecar** | Real SQL execution; Glue-backed views over S3 data; `read_parquet` / `read_json_auto` / `read_csv_auto` inferred from SerDe |+| **Glue** | In-process | Data Catalog; tables consumed by Athena as DuckDB views at query time || **Data Firehose** | In-process | Streaming data delivery; records flushed as NDJSON to S3 || **ECS** | **Real Docker containers** | Clusters, task definitions, tasks, services, capacity providers, task sets || **EC2** | In-process | VPCs, subnets, security groups, instances, AMIs, key pairs, internet gateways, route tables, Elastic IPs, tags |diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.javanew file mode 100644index 0000000000..31e7b031cf--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/AthenaTest.java@@ -0,0 +1,109 @@+package com.floci.test;++import org.junit.jupiter.api.*;+import software.amazon.awssdk.services.athena.AthenaClient;+import software.amazon.awssdk.services.athena.model.*;++import java.time.Duration;++import static org.assertj.core.api.Assertions.*;++@DisplayName("Athena Query Execution")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class AthenaTest {++ private static AthenaClient athena;+ private static String queryExecutionId;++ @BeforeAll+ static void setup() {+ athena = TestFixtures.athenaClient();+ }++ @AfterAll+ static void cleanup() {+ if (athena != null) {+ athena.close();+ }+ }++ @Test+ @Order(1)+ @DisplayName("Start query execution returns an execution ID")+ void startQueryExecution() {+ StartQueryExecutionResponse response = athena.startQueryExecution(+ StartQueryExecutionRequest.builder()+ .queryString("SELECT 1 AS value")+ .workGroup("primary")+ .resultConfiguration(ResultConfiguration.builder()+ .outputLocation("s3://floci-athena-results/sdk-tests/")+ .build())+ .build());++ assertThat(response.queryExecutionId()).isNotBlank();+ queryExecutionId = response.queryExecutionId();+ }++ @Test+ @Order(2)+ @DisplayName("Get query execution returns execution details")+ void getQueryExecution() {+ GetQueryExecutionResponse response = athena.getQueryExecution(+ GetQueryExecutionRequest.builder()+ .queryExecutionId(queryExecutionId)+ .build());++ QueryExecution execution = response.queryExecution();+ assertThat(execution.queryExecutionId()).isEqualTo(queryExecutionId);+ assertThat(execution.query()).isEqualTo("SELECT 1 AS value");+ assertThat(execution.status().state()).isIn(+ QueryExecutionState.RUNNING, QueryExecutionState.SUCCEEDED);+ }++ @Test+ @Order(3)+ @DisplayName("Get query results returns result set")+ void getQueryResults() {+ // Poll until succeeded (mock mode completes immediately, real duck may take a moment)+ QueryExecutionState state = QueryExecutionState.RUNNING;+ int attempts = 0;+ while (state == QueryExecutionState.RUNNING && attempts++ < 20) {+ GetQueryExecutionResponse exec = athena.getQueryExecution(+ GetQueryExecutionRequest.builder().queryExecutionId(queryExecutionId).build());+ state = exec.queryExecution().status().state();+ if (state == QueryExecutionState.RUNNING) {+ try { Thread.sleep(500); } catch (InterruptedException ignored) {}+ }+ }++ assertThat(state).isEqualTo(QueryExecutionState.SUCCEEDED);++ GetQueryResultsResponse results = athena.getQueryResults(+ GetQueryResultsRequest.builder()+ .queryExecutionId(queryExecutionId)+ .build());++ assertThat(results.resultSet()).isNotNull();+ }++ @Test+ @Order(4)+ @DisplayName("List query executions includes started execution")+ void listQueryExecutions() {+ ListQueryExecutionsResponse response = athena.listQueryExecutions(+ ListQueryExecutionsRequest.builder().build());++ assertThat(response.queryExecutionIds()).contains(queryExecutionId);+ }++ @Test+ @Order(5)+ @DisplayName("Get non-existent query execution throws InvalidRequestException")+ void getQueryExecutionNotFound() {+ assertThatThrownBy(() -> athena.getQueryExecution(+ GetQueryExecutionRequest.builder()+ .queryExecutionId("00000000-0000-0000-0000-000000000000")+ .build()))+ .isInstanceOf(InvalidRequestException.class);+ }+}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.javaindex 02682e9dc2..5567ad3d0e 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DataLakeTest.java@@ -12,6 +12,8 @@ import software.amazon.awssdk.services.glue.model.*;import java.nio.charset.StandardCharsets;import java.util.List;+import java.util.Map;+import java.util.stream.Collectors;import static org.assertj.core.api.Assertions.assertThat;@@ -42,14 +44,19 @@ class DataLakeTest {.databaseInput(DatabaseInput.builder().name(DB_NAME).build()).build());- // 2. Glue Table+ // 2. Glue Table — standard AWS JSON table config: TextInputFormat + JsonSerDeglue.createTable(CreateTableRequest.builder().databaseName(DB_NAME).tableInput(TableInput.builder().name(TABLE_NAME).storageDescriptor(StorageDescriptor.builder().location("s3://floci-firehose-results/" + STREAM_NAME + "/")- .inputFormat("Parquet")+ .inputFormat("org.apache.hadoop.mapred.TextInputFormat")+ .outputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat")+ .serdeInfo(SerDeInfo.builder()+ .serializationLibrary("org.openx.data.jsonserde.JsonSerDe")+ .parameters(Map.of("serialization.format", "1"))+ .build()).columns(software.amazon.awssdk.services.glue.model.Column.builder().name("id").type("int").build(),software.amazon.awssdk.services.glue.model.Column.builder().name("amount").type("double").build()@@ -102,13 +109,22 @@ class DataLakeTest {assertThat(status.state()).isEqualTo(QueryExecutionState.SUCCEEDED);- // Get Results — Athena is in mock mode; query execution is accepted and the state- // machine runs correctly, but results are empty until the DuckDB sidecar is integrated.GetQueryResultsResponse results = athena.getQueryResults(GetQueryResultsRequest.builder().queryExecutionId(queryId).build());assertThat(results.resultSet()).isNotNull();- assertThat(results.resultSet().rows()).isEmpty();+ // Athena GetQueryResults includes a header row + data rows+ assertThat(results.resultSet().rows()).hasSizeGreaterThanOrEqualTo(2);++ // Header row must contain the column name+ List<String> header = results.resultSet().rows().get(0).data().stream()+ .map(d -> d.varCharValue())+ .collect(Collectors.toList());+ assertThat(header).containsExactly("total");++ // Data row: sum(amount) = 10+20+30+40+50 = 150+ String total = results.resultSet().rows().get(1).data().get(0).varCharValue();+ assertThat(Double.parseDouble(total)).isEqualTo(150.0);}}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseTest.javanew file mode 100644index 0000000000..75dc42402f--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/FirehoseTest.java@@ -0,0 +1,142 @@+package com.floci.test;++import org.junit.jupiter.api.*;+import software.amazon.awssdk.core.SdkBytes;+import software.amazon.awssdk.services.firehose.FirehoseClient;+import software.amazon.awssdk.services.firehose.model.*;++import java.nio.charset.StandardCharsets;+import java.util.List;+import java.util.UUID;++import static org.assertj.core.api.Assertions.*;++@DisplayName("Firehose Delivery Streams")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class FirehoseTest {++ private static FirehoseClient firehose;+ private static final String STREAM_NAME = "sdk-test-stream-" + UUID.randomUUID().toString().substring(0, 8);++ @BeforeAll+ static void setup() {+ firehose = TestFixtures.firehoseClient();+ }++ @AfterAll+ static void cleanup() {+ if (firehose != null) {+ try {+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME).build());+ } catch (Exception ignored) {}+ firehose.close();+ }+ }++ @Test+ @Order(1)+ @DisplayName("Create delivery stream")+ void createDeliveryStream() {+ CreateDeliveryStreamResponse response = firehose.createDeliveryStream(+ CreateDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .deliveryStreamType(DeliveryStreamType.DIRECT_PUT)+ .s3DestinationConfiguration(S3DestinationConfiguration.builder()+ .bucketARN("arn:aws:s3:::floci-firehose-sdk-test")+ .roleARN("arn:aws:iam::000000000000:role/firehose-role")+ .bufferingHints(BufferingHints.builder()+ .intervalInSeconds(60)+ .sizeInMBs(1)+ .build())+ .build())+ .build());++ assertThat(response.deliveryStreamARN()).contains(STREAM_NAME);+ }++ @Test+ @Order(2)+ @DisplayName("Describe delivery stream")+ void describeDeliveryStream() {+ DescribeDeliveryStreamResponse response = firehose.describeDeliveryStream(+ DescribeDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .build());++ DeliveryStreamDescription desc = response.deliveryStreamDescription();+ assertThat(desc.deliveryStreamName()).isEqualTo(STREAM_NAME);+ assertThat(desc.deliveryStreamStatus()).isEqualTo(DeliveryStreamStatus.ACTIVE);+ assertThat(desc.deliveryStreamARN()).contains(STREAM_NAME);+ }++ @Test+ @Order(3)+ @DisplayName("List delivery streams includes created stream")+ void listDeliveryStreams() {+ ListDeliveryStreamsResponse response = firehose.listDeliveryStreams(+ ListDeliveryStreamsRequest.builder().build());++ assertThat(response.deliveryStreamNames()).contains(STREAM_NAME);+ }++ @Test+ @Order(4)+ @DisplayName("Put single record")+ void putRecord() {+ PutRecordResponse response = firehose.putRecord(PutRecordRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .record(software.amazon.awssdk.services.firehose.model.Record.builder()+ .data(SdkBytes.fromString("{\"event\":\"test\"}", StandardCharsets.UTF_8))+ .build())+ .build());++ assertThat(response.recordId()).isNotBlank();+ }++ @Test+ @Order(5)+ @DisplayName("Put record batch")+ void putRecordBatch() {+ List<software.amazon.awssdk.services.firehose.model.Record> records = List.of(+ software.amazon.awssdk.services.firehose.model.Record.builder().data(SdkBytes.fromString("{\"i\":1}", StandardCharsets.UTF_8)).build(),+ software.amazon.awssdk.services.firehose.model.Record.builder().data(SdkBytes.fromString("{\"i\":2}", StandardCharsets.UTF_8)).build(),+ software.amazon.awssdk.services.firehose.model.Record.builder().data(SdkBytes.fromString("{\"i\":3}", StandardCharsets.UTF_8)).build()+ );++ PutRecordBatchResponse response = firehose.putRecordBatch(PutRecordBatchRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .records(records)+ .build());++ assertThat(response.failedPutCount()).isEqualTo(0);+ assertThat(response.requestResponses()).hasSize(3);+ response.requestResponses().forEach(r -> assertThat(r.recordId()).isNotBlank());+ }++ @Test+ @Order(6)+ @DisplayName("Describe non-existent stream throws ResourceNotFoundException")+ void describeNonExistentStream() {+ assertThatThrownBy(() -> firehose.describeDeliveryStream(+ DescribeDeliveryStreamRequest.builder()+ .deliveryStreamName("nonexistent-stream-xyz")+ .build()))+ .isInstanceOf(ResourceNotFoundException.class);+ }++ @Test+ @Order(7)+ @DisplayName("Delete delivery stream")+ void deleteDeliveryStream() {+ firehose.deleteDeliveryStream(DeleteDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .build());++ assertThatThrownBy(() -> firehose.describeDeliveryStream(+ DescribeDeliveryStreamRequest.builder()+ .deliveryStreamName(STREAM_NAME)+ .build()))+ .isInstanceOf(ResourceNotFoundException.class);+ }+}diff --git a/docs/services/athena.md b/docs/services/athena.mdindex e9ca2855a8..f8f729e7b7 100644--- a/docs/services/athena.md+++ b/docs/services/athena.md@@ -3,15 +3,15 @@**Protocol:** JSON 1.1**Endpoint:** `http://localhost:4566/`-Floci emulates Amazon Athena's control-plane and query state machine. Query execution is in **mock mode** — SQL is accepted and queries immediately transition to `SUCCEEDED` with an empty result set. A dedicated query engine (DuckDB in a sidecar container) will be integrated in a future release.+Floci emulates Amazon Athena with **real SQL execution** powered by a [floci-duck](https://hub.docker.com/r/floci/floci-duck) sidecar container running DuckDB. When a query is submitted, Floci spins up the sidecar on first use, injects `CREATE OR REPLACE VIEW` statements for each Glue-registered table pointing to S3 data, then executes the SQL and stores results as CSV in S3.## Supported Actions| Action | Description ||---|---|-| `StartQueryExecution` | Accepts a SQL query; transitions it to SUCCEEDED immediately |-| `GetQueryExecution` | Returns query status (QUEUED, RUNNING, SUCCEEDED, FAILED) |-| `GetQueryResults` | Returns the result set for a completed query (empty in mock mode) |+| `StartQueryExecution` | Submits a SQL query; executed asynchronously via DuckDB |+| `GetQueryExecution` | Returns query status (`QUEUED`, `RUNNING`, `SUCCEEDED`, `FAILED`) |+| `GetQueryResults` | Returns the result set for a completed query || `ListQueryExecutions` | Returns a list of past query executions || `StopQueryExecution` | Cancels a running query || `CreateWorkGroup` | Creates a new workgroup |@@ -20,25 +20,102 @@ Floci emulates Amazon Athena's control-plane and query state machine. Query exec## How it works-1. **State machine**: Floci implements the full Athena execution state lifecycle (QUEUED → RUNNING → SUCCEEDED/FAILED).-2. **Mock execution**: Queries are accepted but not executed — results are always empty. This allows SDK-based workflow code to be tested end-to-end.+1. **Lazy sidecar start**: On the first `StartQueryExecution` call, Floci checks for a local `floci/floci-duck:latest` image and starts the container. Subsequent queries reuse the running container.+2. **Glue DDL injection**: Floci reads all Glue tables for the target database and generates `CREATE OR REPLACE VIEW` statements mapping each table name to its S3 location via DuckDB's `read_parquet`, `read_json_auto`, or `read_csv_auto` functions — chosen based on the table's `InputFormat` or SerDe serialization library.+3. **Query execution**: The user's SQL is wrapped in `COPY (...) TO 's3://...' (FORMAT CSV, HEADER)` and executed. Results are written directly to the output S3 path.+4. **Results retrieval**: `GetQueryResults` reads the CSV back from S3 and returns it in the standard Athena `ResultSet` shape.-## Example+## Format inference++The DuckDB read function is chosen from the Glue table's `StorageDescriptor`:++| Condition | Read function |+|---|---|+| `InputFormat` or `SerializationLibrary` contains `parquet` | `read_parquet` |+| `InputFormat` or `SerializationLibrary` contains `json` | `read_json_auto` |+| `InputFormat` contains `hive` | `read_json_auto` |+| Anything else | `read_csv_auto` |++## Configuration++| Property | Default | Description |+|---|---|---|+| `FLOCI_SERVICES_ATHENA_MOCK` | `false` | Set to `true` to disable DuckDB execution — queries immediately succeed with empty results |+| `FLOCI_SERVICES_ATHENA_DEFAULT_IMAGE` | `floci/floci-duck:latest` | DuckDB sidecar image |+| `FLOCI_SERVICES_ATHENA_DUCK_URL` | *(unset)* | Point to an existing floci-duck instance and skip container management |++## Example — simple query```bashexport AWS_ENDPOINT_URL=http://localhost:4566# Start a queryQUERY_ID=$(aws athena start-query-execution \- --query-string "SELECT count(*) FROM my_table" \- --query-execution-context Database=my_db \+ --query-string "SELECT 42 AS answer" \--query 'QueryExecutionId' \- --output text \- --endpoint-url $AWS_ENDPOINT_URL)+ --output text)-# Check status-aws athena get-query-execution --query-execution-id $QUERY_ID --endpoint-url $AWS_ENDPOINT_URL+# Wait for completion+aws athena get-query-execution --query-execution-id $QUERY_ID-# Get results (empty in mock mode)-aws athena get-query-results --query-execution-id $QUERY_ID --endpoint-url $AWS_ENDPOINT_URL+# Get results+aws athena get-query-results --query-execution-id $QUERY_ID```++## Example — data lake query (S3 + Glue + Athena)++```bash+export AWS_ENDPOINT_URL=http://localhost:4566++# 1. Create S3 bucket and upload data+aws s3 mb s3://my-data-lake+echo '{"id":1,"amount":10.0}+{"id":2,"amount":20.0}+{"id":3,"amount":30.0}' | aws s3 cp - s3://my-data-lake/orders/data.json++# 2. Register table in Glue+aws glue create-database --database-input '{"Name":"analytics"}'++aws glue create-table \+ --database-name analytics \+ --table-input '{+ "Name": "orders",+ "StorageDescriptor": {+ "Location": "s3://my-data-lake/orders/",+ "InputFormat": "org.apache.hadoop.mapred.TextInputFormat",+ "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",+ "SerdeInfo": {+ "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe"+ },+ "Columns": [+ {"Name": "id", "Type": "int"},+ {"Name": "amount", "Type": "double"}+ ]+ }+ }'++# 3. Run Athena query+QUERY_ID=$(aws athena start-query-execution \+ --query-string "SELECT sum(amount) AS total FROM orders" \+ --query-execution-context Database=analytics \+ --query 'QueryExecutionId' \+ --output text)++# 4. Poll until done+while true; do+ STATE=$(aws athena get-query-execution \+ --query-execution-id $QUERY_ID \+ --query 'QueryExecution.Status.State' \+ --output text)+ [ "$STATE" = "SUCCEEDED" ] && break+ [ "$STATE" = "FAILED" ] && echo "Query failed" && exit 1+ sleep 1+done++# 5. Fetch results+aws athena get-query-results --query-execution-id $QUERY_ID+```++## Mock mode++Set `FLOCI_SERVICES_ATHENA_MOCK=true` to skip DuckDB entirely. In this mode queries transition to `SUCCEEDED` immediately with an empty result set — useful for unit tests that only exercise the Athena state machine, not the query results.diff --git a/docs/services/glue.md b/docs/services/glue.mdindex 51209e7354..a0ebf0c2fe 100644--- a/docs/services/glue.md+++ b/docs/services/glue.md@@ -3,7 +3,7 @@**Protocol:** JSON 1.1**Endpoint:** `http://localhost:4566/`-Floci emulates the AWS Glue Data Catalog, allowing you to manage metadata for your Data Lake locally.+Floci emulates the AWS Glue Data Catalog, allowing you to manage metadata for your data lake locally.## Supported Actions@@ -18,7 +18,16 @@ Floci emulates the AWS Glue Data Catalog, allowing you to manage metadata for yo## Integration with Athena-The Glue Data Catalog is automatically used by **Athena** to resolve table names to S3 locations and formats. When you define a table in Glue, Athena will know how to query the underlying S3 objects.+The Glue Data Catalog is automatically used by **Athena** to resolve table names to S3 locations and formats. When you submit an Athena query, Floci reads all Glue tables for the target database and generates DuckDB views on top of the underlying S3 objects before executing the SQL.++The DuckDB read function is selected based on the table's `StorageDescriptor.InputFormat` and `StorageDescriptor.SerdeInfo.SerializationLibrary`:++| Condition | DuckDB function |+|---|---|+| `InputFormat` or `SerializationLibrary` contains `parquet` | `read_parquet` |+| `InputFormat` or `SerializationLibrary` contains `json` | `read_json_auto` |+| `InputFormat` contains `hive` | `read_json_auto` |+| Anything else | `read_csv_auto` |## Example@@ -26,21 +35,46 @@ The Glue Data Catalog is automatically used by **Athena** to resolve table namesexport AWS_ENDPOINT_URL=http://localhost:4566# Create a database-aws glue create-database --database-input '{"Name": "my_db"}' --endpoint-url $AWS_ENDPOINT_URL+aws glue create-database \+ --database-input '{"Name": "analytics"}' \+ --endpoint-url $AWS_ENDPOINT_URL-# Create a table+# Create a JSON table (standard AWS format for NDJSON data)aws glue create-table \- --database-name my_db \+ --database-name analytics \--table-input '{"Name": "orders","StorageDescriptor": {"Location": "s3://my-bucket/orders/",- "InputFormat": "Parquet",+ "InputFormat": "org.apache.hadoop.mapred.TextInputFormat",+ "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",+ "SerdeInfo": {+ "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe"+ },"Columns": [- {"Name": "id", "Type": "int"},+ {"Name": "id", "Type": "int"},{"Name": "amount", "Type": "double"}]}}' \--endpoint-url $AWS_ENDPOINT_URL++# Create a Parquet table+aws glue create-table \+ --database-name analytics \+ --table-input '{+ "Name": "events",+ "StorageDescriptor": {+ "Location": "s3://my-bucket/events/",+ "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",+ "SerdeInfo": {+ "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"+ },+ "Columns": [+ {"Name": "event_id", "Type": "string"},+ {"Name": "ts", "Type": "bigint"}+ ]+ }+ }' \+ --endpoint-url $AWS_ENDPOINT_URL```diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex ea0a8d5a0e..4f4de875a7 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -440,6 +440,15 @@ public interface EmulatorConfig {interface AthenaServiceConfig {@WithDefault("true")boolean enabled();++ @WithDefault("false")+ boolean mock();++ /** When set, Floci uses this URL and skips floci-duck container management. */+ Optional<String> duckUrl();++ @WithDefault("floci/floci-duck:latest")+ String defaultImage();}interface GlueServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaJsonHandler.javaindex bb13ad5217..48950917f3 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaJsonHandler.java@@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.athena.model.QueryExecution;+import io.github.hectorvent.floci.services.athena.model.QueryExecutionContext;+import io.github.hectorvent.floci.services.athena.model.ResultConfiguration;import io.github.hectorvent.floci.services.athena.model.ResultSet;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -28,11 +30,18 @@ public class AthenaJsonHandler {case "StartQueryExecution" -> {String query = request.get("QueryString").asText();String workGroup = request.has("WorkGroup") ? request.get("WorkGroup").asText() : "primary";- String database = "default";- if (request.has("QueryExecutionContext") && request.get("QueryExecutionContext").has("Database")) {- database = request.get("QueryExecutionContext").get("Database").asText();++ QueryExecutionContext context = null;+ if (request.has("QueryExecutionContext")) {+ context = mapper.treeToValue(request.get("QueryExecutionContext"), QueryExecutionContext.class);+ }++ ResultConfiguration resultConfiguration = null;+ if (request.has("ResultConfiguration")) {+ resultConfiguration = mapper.treeToValue(request.get("ResultConfiguration"), ResultConfiguration.class);}- String id = athenaService.startQueryExecution(query, workGroup, database);++ String id = athenaService.startQueryExecution(query, workGroup, context, resultConfiguration);yield Response.ok(Map.of("QueryExecutionId", id)).build();}case "GetQueryExecution" -> {@@ -46,8 +55,9 @@ public class AthenaJsonHandler {yield Response.ok(Map.of("ResultSet", results)).build();}case "ListQueryExecutions" -> {- yield Response.ok(Map.of("QueryExecutionIds",- athenaService.listQueryExecutions().stream().map(QueryExecution::getQueryExecutionId).toList())).build();+ yield Response.ok(Map.of("QueryExecutionIds",+ athenaService.listQueryExecutions().stream()+ .map(QueryExecution::getQueryExecutionId).toList())).build();}default -> throw new AwsException("InvalidAction", "Action " + action + " is not supported", 400);};diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.javaindex 49aad22f14..31d7592447 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java@@ -1,14 +1,30 @@package io.github.hectorvent.floci.services.athena;import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.athena.model.*;+import io.github.hectorvent.floci.services.athena.FlociDuckManager;+import io.github.hectorvent.floci.services.glue.GlueService;+import io.github.hectorvent.floci.services.glue.model.Table;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import io.vertx.core.Vertx;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.io.BufferedReader;+import java.io.ByteArrayInputStream;+import java.io.InputStreamReader;+import java.net.URI;+import java.net.http.HttpClient;+import java.net.http.HttpRequest;+import java.net.http.HttpResponse;+import java.nio.charset.StandardCharsets;import java.time.Instant;import java.util.*;@@ -16,31 +32,85 @@ import java.util.*;public class AthenaService {private static final Logger LOG = Logger.getLogger(AthenaService.class);+ private static final String DEFAULT_OUTPUT_BUCKET = "floci-athena-results";private final StorageBackend<String, QueryExecution> queryStore;+ private final FlociDuckManager duckManager;+ private final GlueService glueService;+ private final S3Service s3Service;+ private final EmulatorConfig config;+ private final Vertx vertx;+ private final ObjectMapper mapper;+ private final HttpClient httpClient;@Inject- public AthenaService(StorageFactory storageFactory) {- this.queryStore = storageFactory.create("athena", "queries.json", new TypeReference<Map<String, QueryExecution>>() {});+ public AthenaService(StorageFactory storageFactory,+ FlociDuckManager duckManager,+ GlueService glueService,+ S3Service s3Service,+ EmulatorConfig config,+ Vertx vertx,+ ObjectMapper mapper) {+ this.queryStore = storageFactory.create("athena", "queries.json",+ new TypeReference<Map<String, QueryExecution>>() {});+ this.duckManager = duckManager;+ this.glueService = glueService;+ this.s3Service = s3Service;+ this.config = config;+ this.vertx = vertx;+ this.mapper = mapper;+ this.httpClient = HttpClient.newHttpClient();}- public String startQueryExecution(String query, String workGroup, String database) {+ public String startQueryExecution(String query,+ String workGroup,+ QueryExecutionContext context,+ ResultConfiguration resultConfiguration) {String id = UUID.randomUUID().toString();- QueryExecution execution = new QueryExecution(id, query, workGroup);- queryStore.put(id, execution);+ String database = context != null && context.getDatabase() != null ? context.getDatabase() : "default";++ // Ensure output location has a trailing slash so floci-duck writes into the prefix+ String outputLocation = resolveOutputLocation(resultConfiguration, id);+ ResultConfiguration resolvedResult = new ResultConfiguration(outputLocation);+ QueryExecution execution = new QueryExecution(id, query, workGroup, resolvedResult, context);execution.getStatus().setState(QueryExecutionState.RUNNING);- execution.getStatus().setState(QueryExecutionState.SUCCEEDED);- execution.getStatus().setCompletionDateTime(Instant.now());queryStore.put(id, execution);- LOG.infov("Query {0} accepted (mock mode — no execution engine)", id);+ if (config.services().athena().mock()) {+ execution.getStatus().setState(QueryExecutionState.SUCCEEDED);+ execution.getStatus().setCompletionDateTime(Instant.now());+ queryStore.put(id, execution);+ LOG.infov("Query {0} accepted (mock mode)", id);+ return id;+ }++ // Submit async — caller gets the ID immediately while execution runs in background+ String finalDatabase = database;+ vertx.executeBlocking(() -> {+ String duckUrl = duckManager.ensureReady();+ String setupDdl = buildGlueDdl(finalDatabase);+ callDuck(duckUrl, query, setupDdl, outputLocation, id);+ return null;+ }).onSuccess(v -> {+ execution.getStatus().setState(QueryExecutionState.SUCCEEDED);+ execution.getStatus().setCompletionDateTime(Instant.now());+ queryStore.put(id, execution);+ LOG.infov("Query {0} succeeded", id);+ }).onFailure(e -> {+ execution.getStatus().setState(QueryExecutionState.FAILED);+ execution.getStatus().setStateChangeReason(e.getMessage());+ queryStore.put(id, execution);+ LOG.warnv("Query {0} failed: {1}", id, e.getMessage());+ });+return id;}public QueryExecution getQueryExecution(String id) {return queryStore.get(id)- .orElseThrow(() -> new AwsException("InvalidRequestException", "Query execution not found: " + id, 400));+ .orElseThrow(() -> new AwsException("InvalidRequestException",+ "Query execution not found: " + id, 400));}public List<QueryExecution> listQueryExecutions() {@@ -49,9 +119,228 @@ public class AthenaService {public ResultSet getQueryResults(String id) {QueryExecution execution = getQueryExecution(id);+if (execution.getStatus().getState() != QueryExecutionState.SUCCEEDED) {throw new AwsException("InvalidRequestException", "Query has not succeeded yet", 400);}++ if (config.services().athena().mock()+ || execution.getResultConfiguration() == null+ || execution.getResultConfiguration().getOutputLocation() == null) {+ return new ResultSet(List.of(), new ResultSet.ResultSetMetadata(List.of()));+ }++ return readResultsFromS3(execution.getResultConfiguration().getOutputLocation(), id);+ }++ // ── private helpers ───────────────────────────────────────────────────────++ private String buildGlueDdl(String database) {+ StringBuilder sb = new StringBuilder();+ try {+ List<Table> tables = glueService.getTables(database);+ for (Table table : tables) {+ String location = table.getStorageDescriptor() != null+ ? table.getStorageDescriptor().getLocation()+ : null;+ if (location == null || location.isBlank()) {+ continue;+ }+ String readFn = inferReadFunction(table);+ String normalizedLocation = location.endsWith("/")+ ? location.substring(0, location.length() - 1) : location;+ sb.append("CREATE OR REPLACE VIEW \"")+ .append(table.getName())+ .append("\" AS SELECT * FROM ")+ .append(readFn)+ .append("('").append(normalizedLocation).append("/**');\n");+ }+ } catch (Exception e) {+ LOG.debugv("Could not inject Glue DDL for database {0}: {1}", database, e.getMessage());+ }+ return sb.toString();+ }++ private String inferReadFunction(Table table) {+ if (table.getStorageDescriptor() == null) {+ return "read_csv_auto";+ }+ String format = table.getStorageDescriptor().getInputFormat();+ String serde = table.getStorageDescriptor().getSerdeInfo() != null+ ? table.getStorageDescriptor().getSerdeInfo().getSerializationLibrary()+ : null;+ if (containsIgnoreCase(format, "parquet") || containsIgnoreCase(serde, "parquet")) {+ return "read_parquet";+ }+ if (containsIgnoreCase(format, "json") || containsIgnoreCase(serde, "json")+ || containsIgnoreCase(format, "hive")) {+ return "read_json_auto";+ }+ return "read_csv_auto";+ }++ private static boolean containsIgnoreCase(String str, String sub) {+ return str != null && str.toLowerCase().contains(sub);+ }++ private String resolveOutputLocation(ResultConfiguration rc, String queryId) {+ String base = (rc != null && rc.getOutputLocation() != null && !rc.getOutputLocation().isBlank())+ ? rc.getOutputLocation()+ : "s3://" + DEFAULT_OUTPUT_BUCKET + "/results/";+ return base.endsWith("/") ? base + queryId + "/" : base + "/" + queryId + "/";+ }++ private void callDuck(String duckUrl, String sql, String setupDdl, String outputS3Path, String queryId) {+ try {+ // Ensure the output bucket exists+ String bucket = extractBucket(outputS3Path);+ if (bucket != null) {+ try {+ s3Service.createBucket(bucket, config.defaultRegion());+ } catch (Exception ignored) {}+ }++ Map<String, Object> body = new LinkedHashMap<>();+ body.put("sql", sql);+ if (setupDdl != null && !setupDdl.isBlank()) {+ body.put("setup_sql", setupDdl);+ }+ body.put("s3_endpoint", config.baseUrl());+ body.put("s3_region", config.defaultRegion());+ body.put("s3_access_key", "test");+ body.put("s3_secret_key", "test");+ body.put("s3_url_style", "path");+ body.put("output_s3_path", outputS3Path + "results.csv");++ String json = mapper.writeValueAsString(body);++ HttpRequest request = HttpRequest.newBuilder()+ .uri(URI.create(duckUrl + "/execute"))+ .header("Content-Type", "application/json")+ .POST(HttpRequest.BodyPublishers.ofString(json))+ .build();++ HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());++ if (response.statusCode() != 200) {+ throw new RuntimeException("floci-duck returned HTTP " + response.statusCode()+ + ": " + response.body());+ }+ } catch (RuntimeException e) {+ throw e;+ } catch (Exception e) {+ throw new RuntimeException("Failed to call floci-duck for query " + queryId + ": " + e.getMessage(), e);+ }+ }++ private ResultSet readResultsFromS3(String outputLocation, String queryId) {+ try {+ String bucket = extractBucket(outputLocation);+ String prefix = extractKey(outputLocation);+ if (bucket == null) {+ return emptyResultSet();+ }++ List<S3Object> objects = s3Service.listObjects(bucket, prefix, null, 10);+ Optional<S3Object> csv = objects.stream()+ .filter(o -> o.getKey().endsWith(".csv"))+ .findFirst()+ .map(o -> s3Service.getObject(bucket, o.getKey()));++ if (csv.isEmpty()) {+ return emptyResultSet();+ }++ return parseCsv(csv.get().getData());+ } catch (Exception e) {+ LOG.warnv("Could not read query results for {0}: {1}", queryId, e.getMessage());+ return emptyResultSet();+ }+ }++ private ResultSet parseCsv(byte[] data) {+ List<ResultSet.Row> rows = new ArrayList<>();+ List<ResultSet.ColumnInfo> columns = new ArrayList<>();++ try (BufferedReader reader = new BufferedReader(+ new InputStreamReader(new ByteArrayInputStream(data), StandardCharsets.UTF_8))) {++ String headerLine = reader.readLine();+ if (headerLine == null) {+ return emptyResultSet();+ }++ String[] headers = splitCsv(headerLine);+ for (String h : headers) {+ columns.add(new ResultSet.ColumnInfo(h, "varchar"));+ }++ // Header row is included in GetQueryResults per AWS spec+ rows.add(toRow(headers));++ String line;+ while ((line = reader.readLine()) != null) {+ rows.add(toRow(splitCsv(line)));+ }+ } catch (Exception e) {+ LOG.debugv("CSV parse error: {0}", e.getMessage());+ }++ return new ResultSet(rows, new ResultSet.ResultSetMetadata(columns));+ }++ private ResultSet.Row toRow(String[] values) {+ List<ResultSet.Datum> data = new ArrayList<>();+ for (String v : values) {+ data.add(new ResultSet.Datum(v));+ }+ return new ResultSet.Row(data);+ }++ /** Minimal CSV split — handles quoted fields. */+ private String[] splitCsv(String line) {+ List<String> fields = new ArrayList<>();+ StringBuilder sb = new StringBuilder();+ boolean inQuotes = false;+ for (int i = 0; i < line.length(); i++) {+ char c = line.charAt(i);+ if (c == '"') {+ if (inQuotes && i + 1 < line.length() && line.charAt(i + 1) == '"') {+ sb.append('"');+ i++;+ } else {+ inQuotes = !inQuotes;+ }+ } else if (c == ',' && !inQuotes) {+ fields.add(sb.toString());+ sb.setLength(0);+ } else {+ sb.append(c);+ }+ }+ fields.add(sb.toString());+ return fields.toArray(new String[0]);+ }++ private String extractBucket(String s3Path) {+ if (s3Path == null || !s3Path.startsWith("s3://")) {+ return null;+ }+ String without = s3Path.substring(5);+ int slash = without.indexOf('/');+ return slash < 0 ? without : without.substring(0, slash);+ }++ private String extractKey(String s3Path) {+ if (s3Path == null || !s3Path.startsWith("s3://")) {+ return "";+ }+ String without = s3Path.substring(5);+ int slash = without.indexOf('/');+ return slash < 0 ? "" : without.substring(slash + 1);+ }++ private ResultSet emptyResultSet() {return new ResultSet(List.of(), new ResultSet.ResultSetMetadata(List.of()));}}diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/FlociDuckManager.java b/src/main/java/io/github/hectorvent/floci/services/athena/FlociDuckManager.javanew file mode 100644index 0000000000..2cd4d1ad92--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/athena/FlociDuckManager.java@@ -0,0 +1,136 @@+package io.github.hectorvent.floci.services.athena;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.quarkus.runtime.ShutdownEvent;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.net.HttpURLConnection;+import java.net.URI;+import java.util.Optional;++/**+ * Lazily starts and manages the floci-duck sidecar container.+ *+ * On the first call to {@link #ensureReady()}, Floci pulls the image and starts+ * a named container "floci-duck". Subsequent calls return the cached URL immediately.+ *+ * If {@code floci.services.athena.duck-url} is configured, container management is+ * skipped entirely and that URL is used as-is — useful in Docker Compose setups where+ * the user runs floci-duck as a separate service.+ */+@ApplicationScoped+public class FlociDuckManager {++ private static final Logger LOG = Logger.getLogger(FlociDuckManager.class);+ private static final String CONTAINER_NAME = "floci-duck";+ private static final int DUCK_PORT = 3000;+ private static final int HEALTH_POLL_MAX_MS = 30_000;+ private static final int HEALTH_POLL_INTERVAL_MS = 500;++ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final ContainerDetector containerDetector;+ private final EmulatorConfig config;++ private volatile String resolvedUrl;+ private volatile String containerId;++ @Inject+ public FlociDuckManager(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerDetector containerDetector,+ EmulatorConfig config) {+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.containerDetector = containerDetector;+ this.config = config;+ }++ /**+ * Returns the floci-duck base URL, starting the container on first call if needed.+ * Thread-safe — concurrent callers wait while the first thread does the pull+start.+ */+ public synchronized String ensureReady() {+ if (resolvedUrl != null) {+ return resolvedUrl;+ }++ Optional<String> configured = config.services().athena().duckUrl();+ if (configured.isPresent() && !configured.get().isBlank()) {+ resolvedUrl = configured.get();+ LOG.infov("Using pre-configured floci-duck URL: {0}", resolvedUrl);+ return resolvedUrl;+ }++ startContainer();+ return resolvedUrl;+ }++ private void startContainer() {+ String image = config.services().athena().defaultImage();+ LOG.infov("Starting floci-duck container using image {0}", image);++ lifecycleManager.removeIfExists(CONTAINER_NAME);++ ContainerSpec spec = containerBuilder.newContainer(image)+ .withName(CONTAINER_NAME)+ .withEnv("FLOCI_DUCK_S3_ACCESS_KEY", "test")+ .withEnv("FLOCI_DUCK_S3_SECRET_KEY", "test")+ .withEnv("FLOCI_DUCK_S3_REGION", config.defaultRegion())+ .withPortBinding(DUCK_PORT, DUCK_PORT)+ .withDockerNetwork(config.services().dockerNetwork())+ .withLogRotation()+ .build();++ containerId = lifecycleManager.createAndStart(spec).containerId();++ String url = containerDetector.isRunningInContainer()+ ? "http://" + CONTAINER_NAME + ":" + DUCK_PORT+ : "http://localhost:" + DUCK_PORT;++ LOG.infov("floci-duck container started, waiting for health check at {0}", url);+ waitForHealth(url);++ resolvedUrl = url;+ LOG.infov("floci-duck is ready at {0}", resolvedUrl);+ }++ private void waitForHealth(String baseUrl) {+ String healthUrl = baseUrl + "/health";+ long deadline = System.currentTimeMillis() + HEALTH_POLL_MAX_MS;++ while (System.currentTimeMillis() < deadline) {+ try {+ HttpURLConnection conn = (HttpURLConnection) URI.create(healthUrl).toURL().openConnection();+ conn.setConnectTimeout(2000);+ conn.setReadTimeout(2000);+ if (conn.getResponseCode() == 200) {+ return;+ }+ } catch (Exception ignored) {+ }+ try {+ Thread.sleep(HEALTH_POLL_INTERVAL_MS);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new RuntimeException("Interrupted while waiting for floci-duck", e);+ }+ }+ throw new RuntimeException("floci-duck did not become healthy within " + HEALTH_POLL_MAX_MS + " ms");+ }++ void onStop(@Observes ShutdownEvent event) {+ if (containerId == null) {+ return;+ }+ LOG.info("Stopping floci-duck container");+ lifecycleManager.stopAndRemove(containerId, null);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/model/QueryExecution.java b/src/main/java/io/github/hectorvent/floci/services/athena/model/QueryExecution.javaindex 74f958e3b6..5734d2c248 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/model/QueryExecution.java+++ b/src/main/java/io/github/hectorvent/floci/services/athena/model/QueryExecution.java@@ -14,12 +14,22 @@ public class QueryExecution {@JsonProperty("WorkGroup")private String workGroup;+ @JsonProperty("ResultConfiguration")+ private ResultConfiguration resultConfiguration;++ @JsonProperty("QueryExecutionContext")+ private QueryExecutionContext queryExecutionContext;+public QueryExecution() {}- public QueryExecution(String id, String query, String workGroup) {+ public QueryExecution(String id, String query, String workGroup,+ ResultConfiguration resultConfiguration,+ QueryExecutionContext queryExecutionContext) {this.queryExecutionId = id;this.query = query;this.workGroup = workGroup;this.status = new QueryExecutionStatus(QueryExecutionState.QUEUED);+ this.resultConfiguration = resultConfiguration;+ this.queryExecutionContext = queryExecutionContext;}public String getQueryExecutionId() { return queryExecutionId; }@@ -30,4 +40,8 @@ public class QueryExecution {public void setStatus(QueryExecutionStatus status) { this.status = status; }public String getWorkGroup() { return workGroup; }public void setWorkGroup(String workGroup) { this.workGroup = workGroup; }+ public ResultConfiguration getResultConfiguration() { return resultConfiguration; }+ public void setResultConfiguration(ResultConfiguration resultConfiguration) { this.resultConfiguration = resultConfiguration; }+ public QueryExecutionContext getQueryExecutionContext() { return queryExecutionContext; }+ public void setQueryExecutionContext(QueryExecutionContext queryExecutionContext) { this.queryExecutionContext = queryExecutionContext; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/model/QueryExecutionContext.java b/src/main/java/io/github/hectorvent/floci/services/athena/model/QueryExecutionContext.javanew file mode 100644index 0000000000..086155d6e4--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/athena/model/QueryExecutionContext.java@@ -0,0 +1,21 @@+package io.github.hectorvent.floci.services.athena.model;++import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+public class QueryExecutionContext {++ @JsonProperty("Database")+ private String database;++ @JsonProperty("Catalog")+ private String catalog;++ public QueryExecutionContext() {}++ public String getDatabase() { return database; }+ public void setDatabase(String database) { this.database = database; }+ public String getCatalog() { return catalog; }+ public void setCatalog(String catalog) { this.catalog = catalog; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/model/ResultConfiguration.java b/src/main/java/io/github/hectorvent/floci/services/athena/model/ResultConfiguration.javanew file mode 100644index 0000000000..9c103dc93e--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/athena/model/ResultConfiguration.java@@ -0,0 +1,20 @@+package io.github.hectorvent.floci.services.athena.model;++import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+public class ResultConfiguration {++ @JsonProperty("OutputLocation")+ private String outputLocation;++ public ResultConfiguration() {}++ public ResultConfiguration(String outputLocation) {+ this.outputLocation = outputLocation;+ }++ public String getOutputLocation() { return outputLocation; }+ public void setOutputLocation(String outputLocation) { this.outputLocation = outputLocation; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.javaindex 59c7bca726..1702278470 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseJsonHandler.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.firehose;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;import io.github.hectorvent.floci.services.firehose.model.Record;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -29,7 +30,13 @@ public class FirehoseJsonHandler {return switch (action) {case "CreateDeliveryStream" -> {String name = request.get("DeliveryStreamName").asText();- String arn = firehoseService.createDeliveryStream(name);+ S3Destination s3 = null;+ if (request.has("S3DestinationConfiguration")) {+ s3 = mapper.treeToValue(request.get("S3DestinationConfiguration"), S3Destination.class);+ } else if (request.has("ExtendedS3DestinationConfiguration")) {+ s3 = mapper.treeToValue(request.get("ExtendedS3DestinationConfiguration"), S3Destination.class);+ }+ String arn = firehoseService.createDeliveryStream(name, s3);yield Response.ok(Map.of("DeliveryStreamARN", arn)).build();}case "DescribeDeliveryStream" -> {@@ -38,7 +45,9 @@ public class FirehoseJsonHandler {yield Response.ok(Map.of("DeliveryStreamDescription", desc)).build();}case "ListDeliveryStreams" -> {- yield Response.ok(Map.of("DeliveryStreamNames", firehoseService.listDeliveryStreams(), "HasMoreDeliveryStreams", false)).build();+ yield Response.ok(Map.of(+ "DeliveryStreamNames", firehoseService.listDeliveryStreams(),+ "HasMoreDeliveryStreams", false)).build();}case "DeleteDeliveryStream" -> {String name = request.get("DeliveryStreamName").asText();@@ -53,13 +62,15 @@ public class FirehoseJsonHandler {}case "PutRecordBatch" -> {String name = request.get("DeliveryStreamName").asText();- List<Map<String, String>> responseEntries = new ArrayList<>();+ List<Record> records = new ArrayList<>();for (JsonNode recordNode : request.get("Records")) {- Record record = mapper.treeToValue(recordNode, Record.class);- firehoseService.putRecord(name, record);- responseEntries.add(Map.of("RecordId", UUID.randomUUID().toString()));+ records.add(mapper.treeToValue(recordNode, Record.class));}- yield Response.ok(Map.of("FailedPutCount", 0, "RequestResponses", responseEntries)).build();+ firehoseService.putRecordBatch(name, records);+ List<Map<String, String>> responses = records.stream()+ .map(r -> Map.of("RecordId", UUID.randomUUID().toString()))+ .toList();+ yield Response.ok(Map.of("FailedPutCount", 0, "RequestResponses", responses)).build();}default -> throw new AwsException("InvalidAction", "Action " + action + " is not supported", 400);};diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 9f99abce85..679386937c 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -2,9 +2,11 @@ package io.github.hectorvent.floci.services.firehose;import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription;+import io.github.hectorvent.floci.services.firehose.model.DeliveryStreamDescription.S3Destination;import io.github.hectorvent.floci.services.firehose.model.Record;import io.github.hectorvent.floci.services.s3.S3Service;import jakarta.enterprise.context.ApplicationScoped;@@ -12,6 +14,8 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.nio.charset.StandardCharsets;+import java.time.ZoneOffset;+import java.time.ZonedDateTime;import java.util.*;import java.util.concurrent.ConcurrentHashMap;@@ -19,55 +23,79 @@ import java.util.concurrent.ConcurrentHashMap;public class FirehoseService {private static final Logger LOG = Logger.getLogger(FirehoseService.class);+ private static final String DEFAULT_BUCKET = "floci-firehose-results";+ private static final int DEFAULT_FLUSH_COUNT = 5;private final StorageBackend<String, DeliveryStreamDescription> streamStore;private final Map<String, List<byte[]>> buffers = new ConcurrentHashMap<>();private final S3Service s3Service;+ private final RegionResolver regionResolver;@Inject- public FirehoseService(StorageFactory storageFactory, S3Service s3Service) {- this.streamStore = storageFactory.create("firehose", "streams.json", new TypeReference<Map<String, DeliveryStreamDescription>>() {});+ public FirehoseService(StorageFactory storageFactory, S3Service s3Service, RegionResolver regionResolver) {+ this.streamStore = storageFactory.create("firehose", "streams.json",+ new TypeReference<Map<String, DeliveryStreamDescription>>() {});this.s3Service = s3Service;+ this.regionResolver = regionResolver;}- public String createDeliveryStream(String name) {- String arn = "arn:aws:firehose:us-east-1:000000000000:deliverystream/" + name;- DeliveryStreamDescription description = new DeliveryStreamDescription(name, arn);+ public String createDeliveryStream(String name, S3Destination s3Config) {+ String arn = "arn:aws:firehose:" + regionResolver.getDefaultRegion()+ + ":" + regionResolver.getAccountId() + ":deliverystream/" + name;+ DeliveryStreamDescription description = new DeliveryStreamDescription(name, arn, s3Config);streamStore.put(name, description);buffers.put(name, Collections.synchronizedList(new ArrayList<>()));- LOG.infov("Created Firehose Delivery Stream: {0}", name);+ LOG.infov("Created Firehose delivery stream: {0}", name);return arn;}public DeliveryStreamDescription describeDeliveryStream(String name) {return streamStore.get(name)- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Stream not found: " + name, 400));+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",+ "Delivery stream not found: " + name, 400));}public void deleteDeliveryStream(String name) {- describeDeliveryStream(name); // Checks if it exists and throws if not+ describeDeliveryStream(name);streamStore.delete(name);buffers.remove(name);- LOG.infov("Deleted Firehose Delivery Stream: {0}", name);+ LOG.infov("Deleted Firehose delivery stream: {0}", name);}public List<String> listDeliveryStreams() {- return streamStore.scan(k -> true).stream().map(DeliveryStreamDescription::getDeliveryStreamName).toList();+ return streamStore.scan(k -> true).stream()+ .map(DeliveryStreamDescription::getDeliveryStreamName).toList();}public void putRecord(String streamName, Record record) {- describeDeliveryStream(streamName); // Check exists- buffers.get(streamName).add(record.getData());+ DeliveryStreamDescription stream = describeDeliveryStream(streamName);+ buffers.computeIfAbsent(streamName, k -> Collections.synchronizedList(new ArrayList<>()))+ .add(record.getData());- // Flush every 5 records for observable local feedback- if (buffers.get(streamName).size() >= 5) {- flush(streamName);+ if (buffers.get(streamName).size() >= DEFAULT_FLUSH_COUNT) {+ flush(streamName, stream);+ }+ }++ public void putRecordBatch(String streamName, List<Record> records) {+ DeliveryStreamDescription stream = describeDeliveryStream(streamName);+ List<byte[]> buffer = buffers.computeIfAbsent(+ streamName, k -> Collections.synchronizedList(new ArrayList<>()));+ for (Record r : records) {+ buffer.add(r.getData());+ }+ if (buffer.size() >= DEFAULT_FLUSH_COUNT) {+ flush(streamName, stream);}}public void flush(String streamName) {+ streamStore.get(streamName).ifPresent(stream -> flush(streamName, stream));+ }++ private void flush(String streamName, DeliveryStreamDescription stream) {List<byte[]> buffer = buffers.get(streamName);- if (buffer.isEmpty()) {+ if (buffer == null || buffer.isEmpty()) {return;}@@ -78,23 +106,55 @@ public class FirehoseService {}try {+ String bucket = resolveBucket(stream);+ String prefix = resolvePrefix(stream);+ String key = prefix + UUID.randomUUID() + ".json";++ ensureBucket(bucket);+StringBuilder sb = new StringBuilder();for (byte[] data : toFlush) {- sb.append(new String(data, StandardCharsets.UTF_8)).append("\n");+ sb.append(new String(data, StandardCharsets.UTF_8));+ if (!sb.isEmpty() && sb.charAt(sb.length() - 1) != '\n') {+ sb.append('\n');+ }}- String bucket = "floci-firehose-results";- String key = streamName + "/" + UUID.randomUUID() + ".json";-- try {- s3Service.createBucket(bucket, "us-east-1");- } catch (Exception ignored) {}-- byte[] jsonBytes = sb.toString().getBytes(StandardCharsets.UTF_8);- s3Service.putObject(bucket, key, jsonBytes, "application/x-ndjson", Map.of());- LOG.infov("Flushed {0} records from stream {1} to s3://{2}/{3}", toFlush.size(), streamName, bucket, key);+ byte[] body = sb.toString().getBytes(StandardCharsets.UTF_8);+ s3Service.putObject(bucket, key, body, "application/x-ndjson", Map.of());+ LOG.infov("Flushed {0} records from stream {1} to s3://{2}/{3}",+ toFlush.size(), streamName, bucket, key);} catch (Exception e) {LOG.errorv("Failed to flush Firehose stream {0}: {1}", streamName, e.getMessage());}}++ private String resolveBucket(DeliveryStreamDescription stream) {+ S3Destination s3 = stream.s3Destination();+ if (s3 != null && s3.bucketName() != null) {+ return s3.bucketName();+ }+ return DEFAULT_BUCKET;+ }++ private String resolvePrefix(DeliveryStreamDescription stream) {+ S3Destination s3 = stream.s3Destination();+ String prefix = (s3 != null && s3.getPrefix() != null) ? s3.getPrefix() : stream.getDeliveryStreamName() + "/";++ // Substitute time-based placeholders matching real Firehose+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);+ prefix = prefix+ .replace("{year}", String.format("%04d", now.getYear()))+ .replace("{month}", String.format("%02d", now.getMonthValue()))+ .replace("{day}", String.format("%02d", now.getDayOfMonth()))+ .replace("{hour}", String.format("%02d", now.getHour()));++ return prefix.endsWith("/") ? prefix : prefix + "/";+ }++ private void ensureBucket(String bucket) {+ try {+ s3Service.createBucket(bucket, regionResolver.getDefaultRegion());+ } catch (Exception ignored) {}+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.javaindex 8700b0a24b..c7875a337d 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/model/DeliveryStreamDescription.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonFormat;import com.fasterxml.jackson.annotation.JsonProperty;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;+import java.util.List;@RegisterForReflectionpublic class DeliveryStreamDescription {@@ -16,13 +17,16 @@ public class DeliveryStreamDescription {@JsonProperty("CreateTimestamp")@JsonFormat(shape = JsonFormat.Shape.NUMBER)private Instant createTimestamp;+ @JsonProperty("Destinations")+ private List<Destination> destinations;public DeliveryStreamDescription() {}- public DeliveryStreamDescription(String name, String arn) {+ public DeliveryStreamDescription(String name, String arn, S3Destination s3) {this.deliveryStreamName = name;this.deliveryStreamARN = arn;this.deliveryStreamStatus = DeliveryStreamStatus.ACTIVE;this.createTimestamp = Instant.now();+ this.destinations = List.of(new Destination(s3));}public String getDeliveryStreamName() { return deliveryStreamName; }@@ -33,4 +37,62 @@ public class DeliveryStreamDescription {public void setDeliveryStreamStatus(DeliveryStreamStatus deliveryStreamStatus) { this.deliveryStreamStatus = deliveryStreamStatus; }public Instant getCreateTimestamp() { return createTimestamp; }public void setCreateTimestamp(Instant createTimestamp) { this.createTimestamp = createTimestamp; }+ public List<Destination> getDestinations() { return destinations; }+ public void setDestinations(List<Destination> destinations) { this.destinations = destinations; }++ /** Convenience: returns the first S3 destination, or null if none. */+ public S3Destination s3Destination() {+ if (destinations == null || destinations.isEmpty()) return null;+ return destinations.get(0).getS3DestinationDescription();+ }++ @RegisterForReflection+ public static class Destination {+ @JsonProperty("S3DestinationDescription")+ private S3Destination s3DestinationDescription;++ public Destination() {}+ public Destination(S3Destination s3) { this.s3DestinationDescription = s3; }+ public S3Destination getS3DestinationDescription() { return s3DestinationDescription; }+ public void setS3DestinationDescription(S3Destination s3) { this.s3DestinationDescription = s3; }+ }++ @RegisterForReflection+ public static class S3Destination {+ @JsonProperty("BucketARN")+ private String bucketArn;+ @JsonProperty("Prefix")+ private String prefix;+ @JsonProperty("BufferingHints")+ private BufferingHints bufferingHints;++ public S3Destination() {}+ public String getBucketArn() { return bucketArn; }+ public void setBucketArn(String bucketArn) { this.bucketArn = bucketArn; }+ public String getPrefix() { return prefix; }+ public void setPrefix(String prefix) { this.prefix = prefix; }+ public BufferingHints getBufferingHints() { return bufferingHints; }+ public void setBufferingHints(BufferingHints bufferingHints) { this.bufferingHints = bufferingHints; }++ /** Extracts bucket name from ARN: arn:aws:s3:::my-bucket → my-bucket */+ public String bucketName() {+ if (bucketArn == null) return null;+ int last = bucketArn.lastIndexOf(':');+ return last >= 0 ? bucketArn.substring(last + 1) : bucketArn;+ }+ }++ @RegisterForReflection+ public static class BufferingHints {+ @JsonProperty("SizeInMBs")+ private int sizeInMBs = 5;+ @JsonProperty("IntervalInSeconds")+ private int intervalInSeconds = 300;++ public BufferingHints() {}+ public int getSizeInMBs() { return sizeInMBs; }+ public void setSizeInMBs(int sizeInMBs) { this.sizeInMBs = sizeInMBs; }+ public int getIntervalInSeconds() { return intervalInSeconds; }+ public void setIntervalInSeconds(int intervalInSeconds) { this.intervalInSeconds = intervalInSeconds; }+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.javaindex 4ee49c41cc..e5443bf232 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ImageCacheService.java@@ -42,6 +42,11 @@ public class ImageCacheService {if (pulledImages.contains(imageUri)) {return;}+ if (isLocalImagePresent(imageUri)) {+ pulledImages.add(imageUri);+ LOG.infov("Image already present locally, skipping pull: {0}", imageUri);+ return;+ }LOG.infov("Pulling image: {0}", imageUri);try {dockerClient.pullImageCmd(imageUri)@@ -57,6 +62,18 @@ public class ImageCacheService {}}+ private boolean isLocalImagePresent(String imageUri) {+ try {+ dockerClient.inspectImageCmd(imageUri).exec();+ return true;+ } catch (com.github.dockerjava.api.exception.NotFoundException e) {+ return false;+ } catch (Exception e) {+ LOG.debugv("Could not check local image presence for {0}: {1}", imageUri, e.getMessage());+ return false;+ }+ }+private AuthConfig resolveAuth(String imageUri) {String host = extractRegistryHost(imageUri);for (EmulatorConfig.DockerConfig.RegistryCredential cred : registryCredentials) {diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex e76296ab51..11f9532d04 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -155,6 +155,9 @@ floci:validation-wait-seconds: 0athena:enabled: true+ mock: false+ # duck-url: http://floci-duck:3000 # set to skip container management+ default-image: "floci/floci-duck:latest"glue:enabled: trueses:diff --git a/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.javanew file mode 100644index 0000000000..81dc28975d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/athena/AthenaIntegrationTest.java@@ -0,0 +1,106 @@+package io.github.hectorvent.floci.services.athena;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.*;++@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class AthenaIntegrationTest {++ private static final String CONTENT_TYPE = "application/x-amz-json-1.1";++ private static String queryExecutionId;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void startQueryExecution() {+ String response = given()+ .header("X-Amz-Target", "AmazonAthena.StartQueryExecution")+ .contentType(CONTENT_TYPE)+ .body("""+ {+ "QueryString": "SELECT 1",+ "WorkGroup": "primary"+ }+ """)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("QueryExecutionId", notNullValue())+ .extract().path("QueryExecutionId");++ queryExecutionId = response;+ }++ @Test+ @Order(2)+ void getQueryExecution() {+ given()+ .header("X-Amz-Target", "AmazonAthena.GetQueryExecution")+ .contentType(CONTENT_TYPE)+ .body("{ \"QueryExecutionId\": \"" + queryExecutionId + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("QueryExecution.QueryExecutionId", equalTo(queryExecutionId))+ .body("QueryExecution.Status.State", equalTo("SUCCEEDED"));+ }++ @Test+ @Order(3)+ void getQueryResults() {+ given()+ .header("X-Amz-Target", "AmazonAthena.GetQueryResults")+ .contentType(CONTENT_TYPE)+ .body("{ \"QueryExecutionId\": \"" + queryExecutionId + "\" }")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ResultSet", notNullValue());+ }++ @Test+ @Order(4)+ void listQueryExecutions() {+ given()+ .header("X-Amz-Target", "AmazonAthena.ListQueryExecutions")+ .contentType(CONTENT_TYPE)+ .body("{}")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("QueryExecutionIds", notNullValue())+ .body("QueryExecutionIds", hasItem(queryExecutionId));+ }++ @Test+ @Order(5)+ void getQueryExecutionNotFound() {+ given()+ .header("X-Amz-Target", "AmazonAthena.GetQueryExecution")+ .contentType(CONTENT_TYPE)+ .body("{ \"QueryExecutionId\": \"nonexistent-id\" }")+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidRequestException"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseIntegrationTest.javaindex 5fa4d7613f..ae53d18230 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseIntegrationTest.java@@ -1,6 +1,8 @@package io.github.hectorvent.floci.services.firehose;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.MethodOrderer;import org.junit.jupiter.api.Order;import org.junit.jupiter.api.Test;@@ -15,6 +17,11 @@ class FirehoseIntegrationTest {private static final String STREAM_NAME = "test-delivery-stream";+ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }+@Test@Order(1)void createDeliveryStream() {diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 00c924d9ee..cfb826722e 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -116,6 +116,7 @@ floci:validation-wait-seconds: 0athena:enabled: true+ mock: trueglue:enabled: trueses: