Light, fluffy, and always free - The AWS Local Emulator alternative
feat(s3): full SelectObjectContent evaluator with floci-duck integration (#835)
Hector Venturae725f38ac755
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3SelectTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3SelectTest.javanew file mode 100644index 0000000000..c98cc819d6--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3SelectTest.java@@ -0,0 +1,301 @@+package com.floci.test;++import org.junit.jupiter.api.*;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.core.async.SdkPublisher;+import software.amazon.awssdk.core.sync.RequestBody;+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;+import software.amazon.awssdk.http.Protocol;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.s3.S3AsyncClient;+import software.amazon.awssdk.services.s3.S3Client;+import software.amazon.awssdk.services.s3.model.*;++import java.io.ByteArrayOutputStream;+import java.nio.charset.StandardCharsets;+import java.util.concurrent.CompletableFuture;++import static org.assertj.core.api.Assertions.*;++@DisplayName("S3 SelectObjectContent")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class S3SelectTest {++ private static S3Client s3;+ private static S3AsyncClient s3Async;+ private static final String BUCKET = "compat-select-bucket";++ @BeforeAll+ static void setup() {+ s3 = TestFixtures.s3Client();+ s3Async = S3AsyncClient.builder()+ .endpointOverride(TestFixtures.endpoint())+ .region(Region.US_EAST_1)+ .credentialsProvider(StaticCredentialsProvider.create(+ AwsBasicCredentials.create("test", "test")))+ .forcePathStyle(true)+ .httpClientBuilder(NettyNioAsyncHttpClient.builder().protocol(Protocol.HTTP1_1))+ .build();+ try {+ s3.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build());+ } catch (BucketAlreadyOwnedByYouException ignored) {}+ }++ @AfterAll+ static void cleanup() {+ if (s3 == null) return;+ try {+ ListObjectsV2Response list = s3.listObjectsV2(+ ListObjectsV2Request.builder().bucket(BUCKET).build());+ for (S3Object obj : list.contents()) {+ s3.deleteObject(DeleteObjectRequest.builder()+ .bucket(BUCKET).key(obj.key()).build());+ }+ s3.deleteBucket(DeleteBucketRequest.builder().bucket(BUCKET).build());+ } catch (Exception ignored) {}+ s3Async.close();+ s3.close();+ }++ // ── helpers ───────────────────────────────────────────────────────────────++ private void put(String key, String body) {+ s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(key).build(),+ RequestBody.fromString(body));+ }++ private static final InputSerialization CSV_USE = InputSerialization.builder()+ .csv(CSVInput.builder().fileHeaderInfo(FileHeaderInfo.USE).build()).build();+ private static final InputSerialization CSV_NONE = InputSerialization.builder()+ .csv(CSVInput.builder().fileHeaderInfo(FileHeaderInfo.NONE).build()).build();+ private static final InputSerialization CSV_IGNORE = InputSerialization.builder()+ .csv(CSVInput.builder().fileHeaderInfo(FileHeaderInfo.IGNORE).build()).build();+ private static final InputSerialization JSON_IN = InputSerialization.builder()+ .json(JSONInput.builder().type(JSONType.LINES).build()).build();+ private static final OutputSerialization CSV_OUT = OutputSerialization.builder()+ .csv(CSVOutput.builder().build()).build();+ private static final OutputSerialization JSON_OUT = OutputSerialization.builder()+ .json(JSONOutput.builder().build()).build();++ private String select(String key, String expression,+ InputSerialization in, OutputSerialization out) {+ ByteArrayOutputStream baos = new ByteArrayOutputStream();+ CompletableFuture<Void> future = s3Async.selectObjectContent(+ SelectObjectContentRequest.builder()+ .bucket(BUCKET).key(key)+ .expression(expression)+ .expressionType(ExpressionType.SQL)+ .inputSerialization(in)+ .outputSerialization(out)+ .build(),+ SelectObjectContentResponseHandler.builder()+ .subscriber(SelectObjectContentResponseHandler.Visitor.builder()+ .onRecords(event -> {+ try {+ baos.write(event.payload().asByteArray());+ } catch (Exception e) {+ throw new RuntimeException(e);+ }+ })+ .build())+ .build());+ future.join();+ return baos.toString(StandardCharsets.UTF_8);+ }++ // ── CSV WHERE clause ───────────────────────────────────────────────────────++ @Test+ @Order(1)+ @DisplayName("CSV: WHERE with numeric comparison")+ void csvWhereNumeric() {+ put("where-num.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ String result = select("where-num.csv", "SELECT * FROM S3Object WHERE age > 30", CSV_USE, CSV_OUT);+ assertThat(result).contains("Charlie").doesNotContain("Alice").doesNotContain("Bob");+ }++ @Test+ @Order(2)+ @DisplayName("CSV: column projection")+ void csvProjection() {+ put("proj.csv", "name,age,city\nAlice,30,New York\nBob,25,London");+ String result = select("proj.csv", "SELECT name, city FROM S3Object", CSV_USE, CSV_OUT);+ assertThat(result).contains("Alice,New York").contains("Bob,London")+ .doesNotContain("30").doesNotContain("25");+ }++ @Test+ @Order(3)+ @DisplayName("CSV: LIMIT")+ void csvLimit() {+ put("limit.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ String result = select("limit.csv", "SELECT * FROM S3Object LIMIT 2", CSV_USE, CSV_OUT);+ assertThat(result).contains("Alice").contains("Bob").doesNotContain("Charlie");+ }++ // ── String equality / case sensitivity ───────────────────────────────────++ @Test+ @Order(4)+ @DisplayName("CSV: string equality matches exact case")+ void csvStringEquality() {+ put("case.csv", "name,age\nAlice,30\nBob,25");+ String result = select("case.csv", "SELECT * FROM S3Object WHERE name = 'Alice'", CSV_USE, CSV_OUT);+ assertThat(result).contains("Alice").doesNotContain("Bob");+ }++ @Test+ @Order(5)+ @DisplayName("CSV: string equality is case-sensitive")+ void csvStringEqualityCaseSensitive() {+ put("case2.csv", "name,age\nAlice,30\nBob,25");+ String result = select("case2.csv", "SELECT * FROM S3Object WHERE name = 'alice'", CSV_USE, CSV_OUT);+ assertThat(result).doesNotContain("Alice").doesNotContain("Bob");+ }++ // ── FileHeaderInfo modes ──────────────────────────────────────────────────++ @Test+ @Order(6)+ @DisplayName("CSV: FileHeaderInfo=NONE treats header line as data")+ void csvFileHeaderInfoNone() {+ put("none.csv", "name,age\nAlice,30\nBob,25");+ String result = select("none.csv", "SELECT _1 FROM S3Object", CSV_NONE, CSV_OUT);+ assertThat(result).contains("name").contains("Alice").contains("Bob");+ }++ @Test+ @Order(7)+ @DisplayName("CSV: FileHeaderInfo=IGNORE skips header row")+ void csvFileHeaderInfoIgnore() {+ put("ignore.csv", "name,age\nAlice,30\nBob,25");+ String result = select("ignore.csv", "SELECT _1 FROM S3Object", CSV_IGNORE, CSV_OUT);+ assertThat(result).contains("Alice").contains("Bob").doesNotContain("name");+ }++ // ── Quoted fields ─────────────────────────────────────────────────────────++ @Test+ @Order(8)+ @DisplayName("CSV: quoted field with embedded comma is not split")+ void csvQuotedField() {+ put("quoted.csv", "name,age\n\"Smith, John\",35\n\"Doe, Jane\",22");+ String result = select("quoted.csv", "SELECT * FROM S3Object WHERE age > 30", CSV_USE, CSV_OUT);+ assertThat(result).contains("Smith, John").doesNotContain("Doe, Jane");+ }++ // ── Enhanced WHERE operators ──────────────────────────────────────────────++ @Test+ @Order(9)+ @DisplayName("CSV: WHERE AND")+ void csvWhereAnd() {+ put("and.csv", "name,age\nAlice,30\nBob,20\nCharlie,35");+ String result = select("and.csv", "SELECT * FROM S3Object WHERE age > 20 AND age < 35", CSV_USE, CSV_OUT);+ assertThat(result).contains("Alice").doesNotContain("Bob").doesNotContain("Charlie");+ }++ @Test+ @Order(10)+ @DisplayName("CSV: WHERE OR")+ void csvWhereOr() {+ put("or.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ String result = select("or.csv",+ "SELECT * FROM S3Object WHERE name = 'Alice' OR name = 'Charlie'", CSV_USE, CSV_OUT);+ assertThat(result).contains("Alice").contains("Charlie").doesNotContain("Bob");+ }++ @Test+ @Order(11)+ @DisplayName("CSV: WHERE LIKE")+ void csvWhereLike() {+ put("like.csv", "name,age\nAlice,30\nBob,25\nAlex,28");+ String result = select("like.csv", "SELECT * FROM S3Object WHERE name LIKE 'Al%'", CSV_USE, CSV_OUT);+ assertThat(result).contains("Alice").contains("Alex").doesNotContain("Bob");+ }++ @Test+ @Order(12)+ @DisplayName("CSV: WHERE BETWEEN")+ void csvWhereBetween() {+ put("between.csv", "name,age\nAlice,30\nBob,20\nCharlie,25");+ String result = select("between.csv",+ "SELECT * FROM S3Object WHERE age BETWEEN 25 AND 30", CSV_USE, CSV_OUT);+ assertThat(result).contains("Alice").contains("Charlie").doesNotContain("Bob");+ }++ @Test+ @Order(13)+ @DisplayName("CSV: WHERE IN")+ void csvWhereIn() {+ put("in.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ String result = select("in.csv",+ "SELECT * FROM S3Object WHERE name IN ('Alice', 'Charlie')", CSV_USE, CSV_OUT);+ assertThat(result).contains("Alice").contains("Charlie").doesNotContain("Bob");+ }++ @Test+ @Order(14)+ @DisplayName("CSV: WHERE IS NULL")+ void csvWhereIsNull() {+ put("isnull.csv", "name,age,city\nAlice,30,NY\nBob,25");+ String result = select("isnull.csv",+ "SELECT name FROM S3Object WHERE city IS NULL", CSV_USE, CSV_OUT);+ assertThat(result).contains("Bob").doesNotContain("Alice");+ }++ // ── JSON input ────────────────────────────────────────────────────────────++ @Test+ @Order(15)+ @DisplayName("JSON Lines: WHERE clause")+ void jsonLinesWhere() {+ put("where.json",+ "{\"name\":\"Alice\",\"age\":30}\n{\"name\":\"Bob\",\"age\":25}\n{\"name\":\"Charlie\",\"age\":35}");+ String result = select("where.json", "SELECT * FROM S3Object WHERE age > 30", JSON_IN, JSON_OUT);+ assertThat(result).contains("Charlie").doesNotContain("Alice").doesNotContain("Bob");+ }++ @Test+ @Order(16)+ @DisplayName("JSON Lines: projection")+ void jsonLinesProjection() {+ put("proj.json",+ "{\"name\":\"Alice\",\"age\":30,\"city\":\"NY\"}\n{\"name\":\"Bob\",\"age\":25,\"city\":\"LA\"}");+ String result = select("proj.json", "SELECT name FROM S3Object", JSON_IN, JSON_OUT);+ assertThat(result).contains("Alice").contains("Bob")+ .doesNotContain("\"age\"").doesNotContain("\"city\"");+ }++ @Test+ @Order(17)+ @DisplayName("JSON Lines: IS NULL")+ void jsonLinesIsNull() {+ put("null.json",+ "{\"name\":\"Alice\",\"age\":30,\"city\":\"NY\"}\n{\"name\":\"Bob\",\"age\":25}");+ String result = select("null.json",+ "SELECT name FROM S3Object WHERE city IS NULL", JSON_IN, JSON_OUT);+ assertThat(result).contains("Bob").doesNotContain("Alice");+ }++ // ── Cross-format output ───────────────────────────────────────────────────++ @Test+ @Order(18)+ @DisplayName("CSV input → JSON output")+ void csvInputJsonOutput() {+ put("xfmt.csv", "name,age\nAlice,30\nBob,25");+ String result = select("xfmt.csv", "SELECT * FROM S3Object", CSV_USE, JSON_OUT);+ assertThat(result).contains("\"name\"").contains("\"Alice\"").contains("\"Bob\"");+ }++ @Test+ @Order(19)+ @DisplayName("JSON input → CSV output")+ void jsonInputCsvOutput() {+ put("xfmt.json", "{\"name\":\"Alice\",\"age\":30}\n{\"name\":\"Bob\",\"age\":25}");+ String result = select("xfmt.json", "SELECT name, age FROM S3Object", JSON_IN, CSV_OUT);+ assertThat(result).contains("Alice").contains("Bob");+ }+}diff --git a/docs/services/athena.md b/docs/services/athena.mdindex f8f729e7b7..f96827a50b 100644--- a/docs/services/athena.md+++ b/docs/services/athena.md@@ -41,8 +41,8 @@ The DuckDB read function is chosen from the Glue table's `StorageDescriptor`:| Property | Default | Description ||---|---|---|| `FLOCI_SERVICES_ATHENA_MOCK` | `false` | Set to `true` to disable DuckDB execution — queries immediately succeed with empty results |-| `FLOCI_SERVICES_ATHENA_DEFAULT_IMAGE` | `floci/floci-duck:latest` | DuckDB sidecar image |-| `FLOCI_SERVICES_ATHENA_DUCK_URL` | *(unset)* | Point to an existing floci-duck instance and skip container management |+| `FLOCI_SERVICES_DUCK_DEFAULT_IMAGE` | `floci/floci-duck:latest` | DuckDB sidecar image pulled on first use |+| `FLOCI_SERVICES_DUCK_URL` | *(unset)* | Point to an existing floci-duck instance and skip container management |## Example — simple query@@ -116,6 +116,14 @@ doneaws athena get-query-results --query-execution-id $QUERY_ID```+## Shared sidecar with S3 Select++The floci-duck sidecar is shared between Athena and S3 Select. Once started by the first Athena query, it is also used by `SelectObjectContent` for CSV (with `FileHeaderInfo=USE`), JSON, and Parquet inputs. If Athena has not yet executed a query, S3 Select falls back to the built-in Java evaluator for CSV and JSON — Parquet always requires the sidecar.++See [S3 Select](s3.md#s3-select) for details on execution modes and supported SQL operators.+## Mock mode-Set `FLOCI_SERVICES_ATHENA_MOCK=true` to skip DuckDB entirely. In this mode queries transition to `SUCCEEDED` immediately with an empty result set — useful for unit tests that only exercise the Athena state machine, not the query results.+Set `FLOCI_SERVICES_ATHENA_MOCK=true` to skip DuckDB entirely for Athena. In this mode queries transition to `SUCCEEDED` immediately with an empty result set — useful for unit tests that only exercise the Athena state machine, not the query results.++When mock mode is enabled the sidecar does **not** start. S3 Select will use the Java evaluator for CSV and JSON. Parquet queries will fail unless `FLOCI_SERVICES_DUCK_URL` points to an already-running floci-duck instance.diff --git a/docs/services/s3.md b/docs/services/s3.mdindex c7a9f3e8c6..a57423dcdf 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -26,6 +26,103 @@**RestoreObject** is accepted but stubbed: Floci validates the request and returns `202 Accepted`, but no restore state machine runs.+## S3 Select++`SelectObjectContent` runs SQL queries directly against S3 objects without downloading the entire file. Floci supports CSV, JSON Lines, JSON arrays, and Parquet inputs. The SQL dialect follows [AWS S3 Select SQL reference](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-select-sql-reference.html).++### Execution modes++Floci chooses the execution engine automatically based on the input format and whether the [floci-duck](https://hub.docker.com/r/floci/floci-duck) sidecar is running:++| Condition | Engine | Notes |+|---|---|---|+| Input is Parquet | floci-duck (required) | DuckDB's `read_parquet` — sidecar must be available |+| Input is CSV with `FileHeaderInfo=USE` and floci-duck is running | floci-duck | Full DuckDB SQL: all operators, LIKE, BETWEEN, IN, IS NULL, AND/OR/NOT |+| Input is JSON and floci-duck is running | floci-duck | `read_json_auto` — supports JSON Lines and JSON arrays |+| Input is CSV with `FileHeaderInfo=NONE` or `IGNORE`, or floci-duck is not running | Java evaluator | Supports SELECT *, column projection, simple WHERE with =, !=, <, >, <=, >=, LIKE, BETWEEN, IN, IS NULL, AND/OR/NOT, LIMIT |++The floci-duck sidecar starts lazily on the **first Athena query**. Until then, `isAvailable()` returns false and S3 Select falls back to the Java evaluator for CSV and JSON. Once the sidecar is running, subsequent S3 Select calls route through DuckDB automatically.++If floci-duck is not running and the object is Parquet, S3 Select returns an error — Parquet decoding requires DuckDB.++### FileHeaderInfo modes (CSV)++| Value | Behavior |+|---|---|+| `USE` | First row is the header; column names are available in WHERE and SELECT |+| `IGNORE` | First row is skipped and not included in output; only positional `_N` references work |+| `NONE` | All rows are data; only positional `_N` references work (e.g. `WHERE _1 = 'Alice'`) |++### Supported SQL operators++When using the Java evaluator (no floci-duck, or CSV with `FileHeaderInfo=NONE/IGNORE`):++- Comparison: `=`, `!=`, `<>`, `<`, `>`, `<=`, `>=`+- Pattern matching: `LIKE` (supports `%` and `_` wildcards)+- Range: `BETWEEN ... AND ...`+- Set membership: `IN (...)`+- Null checks: `IS NULL`, `IS NOT NULL`+- Logical: `AND`, `OR`, `NOT`+- Clauses: `SELECT *`, column projection, `LIMIT`++### Output formats++S3 Select supports cross-format output: a CSV object can produce JSON output and vice versa.++| Input | Output | Format |+|---|---|---|+| CSV | CSV | Default — comma-separated values |+| CSV | JSON | One JSON object per row: `{"col1":"val1","col2":"val2"}` |+| JSON | JSON | Default — one JSON object per line |+| JSON | CSV | Comma-separated values, values quoted when they contain commas or newlines |++### Example++```bash+export AWS_ENDPOINT_URL=http://localhost:4566++# Upload a CSV file+printf 'name,age,city\nAlice,30,New York\nBob,25,\nCharlie,35,London\n' \+ | aws s3 cp - s3://my-bucket/people.csv++# Query with WHERE and column projection+aws s3api select-object-content \+ --bucket my-bucket \+ --key people.csv \+ --expression "SELECT name, city FROM S3Object WHERE age >= 30" \+ --expression-type SQL \+ --input-serialization '{"CSV":{"FileHeaderInfo":"USE"}}' \+ --output-serialization '{"CSV":{}}' \+ /dev/stdout++# IS NULL check+aws s3api select-object-content \+ --bucket my-bucket \+ --key people.csv \+ --expression "SELECT name FROM S3Object WHERE city IS NULL" \+ --expression-type SQL \+ --input-serialization '{"CSV":{"FileHeaderInfo":"USE"}}' \+ --output-serialization '{"CSV":{}}' \+ /dev/stdout++# JSON Lines input+printf '{"name":"Alice","score":95}\n{"name":"Bob","score":72}\n' \+ | aws s3 cp - s3://my-bucket/scores.json++aws s3api select-object-content \+ --bucket my-bucket \+ --key scores.json \+ --expression "SELECT * FROM S3Object WHERE score > 80" \+ --expression-type SQL \+ --input-serialization '{"JSON":{"Type":"LINES"}}' \+ --output-serialization '{"JSON":{}}' \+ /dev/stdout+```++### Mock mode note++When `FLOCI_SERVICES_ATHENA_MOCK=true` is set, Athena queries are stubbed but floci-duck does **not** start. In that configuration, S3 Select uses the Java evaluator for CSV and JSON. Parquet queries will fail unless `FLOCI_SERVICES_DUCK_URL` points to an already-running floci-duck instance.+## Not ImplementedThese AWS S3 features have no handler in Floci. Calls will return an error (typically `404` or `NoSuchBucket`-style):diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex da4f07d859..81a2cda437 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -301,6 +301,7 @@ public interface EmulatorConfig {TransferServiceConfig transfer();TextractServiceConfig textract();PricingServiceConfig pricing();+ DuckConfig duck();TranscribeServiceConfig transcribe();CostExplorerServiceConfig ce();}@@ -552,9 +553,11 @@ public interface EmulatorConfig {@WithDefault("false")boolean mock();+ }+ interface DuckConfig {/** When set, Floci uses this URL and skips floci-duck container management. */- Optional<String> duckUrl();+ Optional<String> url();@WithDefault("floci/floci-duck:latest")String defaultImage();diff --git a/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java b/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.javanew file mode 100644index 0000000000..ba993e26eb--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java@@ -0,0 +1,51 @@+package io.github.hectorvent.floci.core.common;++import java.util.ArrayList;+import java.util.List;++public class CsvParser {++ private CsvParser() {}++ public static List<String> parseLine(String line) {+ List<String> fields = new ArrayList<>();+ StringBuilder current = new StringBuilder();+ boolean inQuotes = false;+ for (int i = 0; i < line.length(); i++) {+ char c = line.charAt(i);+ if (inQuotes) {+ if (c == '"') {+ if (i + 1 < line.length() && line.charAt(i + 1) == '"') {+ current.append('"');+ i++;+ } else {+ inQuotes = false;+ }+ } else {+ current.append(c);+ }+ } else {+ if (c == '"') {+ inQuotes = true;+ } else if (c == ',') {+ fields.add(current.toString());+ current.setLength(0);+ } else {+ current.append(c);+ }+ }+ }+ fields.add(current.toString());+ return fields;+ }++ public static List<List<String>> parseAll(String content) {+ List<List<String>> rows = new ArrayList<>();+ for (String line : content.split("\\r?\\n")) {+ if (!line.trim().isEmpty()) {+ rows.add(parseLine(line));+ }+ }+ return rows;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.javaindex 65c83497a1..eac6828dfd 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java@@ -1,14 +1,13 @@package io.github.hectorvent.floci.services.athena;import com.fasterxml.jackson.core.type.TypeReference;-import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;-import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;-import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import io.github.hectorvent.floci.core.common.CsvParser;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.athena.model.*;+import io.github.hectorvent.floci.services.floci.FlociDuckClient;import io.github.hectorvent.floci.services.glue.GlueService;import io.github.hectorvent.floci.services.glue.model.Table;import io.github.hectorvent.floci.services.s3.S3Service;@@ -21,10 +20,6 @@ import org.jboss.logging.Logger;import java.io.BufferedReader;import java.io.ByteArrayInputStream;import java.io.InputStreamReader;-import java.net.URI;-import java.net.http.HttpClient;-import java.net.http.HttpRequest;-import java.net.http.HttpResponse;import java.nio.charset.StandardCharsets;import java.time.Instant;import java.util.*;@@ -36,37 +31,26 @@ public class AthenaService {private static final String DEFAULT_OUTPUT_BUCKET = "floci-athena-results";private final StorageBackend<String, QueryExecution> queryStore;- private final FlociDuckManager duckManager;+ private final FlociDuckClient duckClient;private final GlueService glueService;private final S3Service s3Service;private final EmulatorConfig config;private final Vertx vertx;- private final ObjectMapper mapper;- private final HttpClient httpClient;- private final DockerHostResolver dockerHostResolver;- private final EmbeddedDnsServer embeddedDnsServer;@Injectpublic AthenaService(StorageFactory storageFactory,- FlociDuckManager duckManager,+ FlociDuckClient duckClient,GlueService glueService,S3Service s3Service,EmulatorConfig config,- Vertx vertx,- ObjectMapper mapper,- DockerHostResolver dockerHostResolver,- EmbeddedDnsServer embeddedDnsServer) {+ Vertx vertx) {this.queryStore = storageFactory.create("athena", "queries.json",- new TypeReference<Map<String, QueryExecution>>() {});- this.duckManager = duckManager;+ new TypeReference<>() {});+ this.duckClient = duckClient;this.glueService = glueService;this.s3Service = s3Service;this.config = config;this.vertx = vertx;- this.mapper = mapper;- this.httpClient = HttpClient.newHttpClient();- this.dockerHostResolver = dockerHostResolver;- this.embeddedDnsServer = embeddedDnsServer;}public String startQueryExecution(String query,@@ -95,9 +79,9 @@ public class AthenaService {// Submit async — caller gets the ID immediately while execution runs in backgroundString finalDatabase = database;vertx.executeBlocking(() -> {- String duckUrl = duckManager.ensureReady();String setupDdl = buildGlueDdl(finalDatabase);- callDuck(duckUrl, query, setupDdl, outputLocation, id);+ ensureOutputBucket(outputLocation);+ duckClient.execute(query, setupDdl, outputLocation + "results.csv");return null;}).onSuccess(v -> {execution.getStatus().setState(QueryExecutionState.SUCCEEDED);@@ -197,56 +181,12 @@ public class AthenaService {return base.endsWith("/") ? base + queryId + "/" : base + "/" + queryId + "/";}- private void callDuck(String duckUrl, String sql, String setupDdl, String outputS3Path, String queryId) {- try {- // Ensure the output bucket exists- String bucket = extractBucket(outputS3Path);- if (bucket != null) {- try {- s3Service.createBucket(bucket, config.defaultRegion());- } catch (Exception ignored) {}- }-- // Floci endpoint reachable from inside the floci-duck container.- // When the embedded DNS server is active, floci-duck containers already have it wired as their- // resolver and can reach Floci by the configured hostname (or the default DNS suffix).- // Fall back to the raw Docker host IP when the embedded DNS is not running (local dev mode).- int flociPort = URI.create(config.baseUrl()).getPort();- String flociHostname = embeddedDnsServer.getServerIp().isPresent()- ? config.hostname().orElse(EmbeddedDnsServer.DEFAULT_SUFFIX)- : dockerHostResolver.resolve();- String flociEndpoint = "http://" + flociHostname + ":" + flociPort;-- Map<String, Object> body = new LinkedHashMap<>();- body.put("sql", sql);- if (setupDdl != null && !setupDdl.isBlank()) {- body.put("setup_sql", setupDdl);- }- body.put("s3_endpoint", flociEndpoint);- body.put("s3_region", config.defaultRegion());- body.put("s3_access_key", "test");- body.put("s3_secret_key", "test");- body.put("s3_url_style", "path");- body.put("output_s3_path", outputS3Path + "results.csv");-- String json = mapper.writeValueAsString(body);-- HttpRequest request = HttpRequest.newBuilder()- .uri(URI.create(duckUrl + "/execute"))- .header("Content-Type", "application/json")- .POST(HttpRequest.BodyPublishers.ofString(json))- .build();-- HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());-- if (response.statusCode() != 200) {- throw new RuntimeException("floci-duck returned HTTP " + response.statusCode()- + ": " + response.body());- }- } catch (RuntimeException e) {- throw e;- } catch (Exception e) {- throw new RuntimeException("Failed to call floci-duck for query " + queryId + ": " + e.getMessage(), e);+ private void ensureOutputBucket(String s3Path) {+ String bucket = extractBucket(s3Path);+ if (bucket != null) {+ try {+ s3Service.createBucket(bucket, config.defaultRegion());+ } catch (Exception ignored) {}}}@@ -287,7 +227,7 @@ public class AthenaService {return emptyResultSet();}- String[] headers = splitCsv(headerLine);+ String[] headers = CsvParser.parseLine(headerLine).toArray(String[]::new);for (String h : headers) {columns.add(new ResultSet.ColumnInfo(h, "varchar"));}@@ -297,7 +237,7 @@ public class AthenaService {String line;while ((line = reader.readLine()) != null) {- rows.add(toRow(splitCsv(line)));+ rows.add(toRow(CsvParser.parseLine(line).toArray(String[]::new)));}} catch (Exception e) {LOG.debugv("CSV parse error: {0}", e.getMessage());@@ -314,31 +254,6 @@ public class AthenaService {return new ResultSet.Row(data);}- /** Minimal CSV split — handles quoted fields. */- private String[] splitCsv(String line) {- List<String> fields = new ArrayList<>();- StringBuilder sb = new StringBuilder();- boolean inQuotes = false;- for (int i = 0; i < line.length(); i++) {- char c = line.charAt(i);- if (c == '"') {- if (inQuotes && i + 1 < line.length() && line.charAt(i + 1) == '"') {- sb.append('"');- i++;- } else {- inQuotes = !inQuotes;- }- } else if (c == ',' && !inQuotes) {- fields.add(sb.toString());- sb.setLength(0);- } else {- sb.append(c);- }- }- fields.add(sb.toString());- return fields.toArray(new String[0]);- }-private String extractBucket(String s3Path) {if (s3Path == null || !s3Path.startsWith("s3://")) {return null;diff --git a/src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckClient.java b/src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckClient.javanew file mode 100644index 0000000000..6b7c430de8--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckClient.java@@ -0,0 +1,164 @@+package io.github.hectorvent.floci.services.floci;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;++import java.net.URI;+import java.net.http.HttpClient;+import java.net.http.HttpRequest;+import java.net.http.HttpResponse;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++/**+ * HTTP client for the floci-duck sidecar.+ *+ * Wraps the two duck endpoints used across services:+ * - /execute — fire-and-forget SQL, writes results to S3 (used by Athena)+ * - /query — SQL that returns rows as JSON maps (used by S3 Select)+ */+@ApplicationScoped+public class FlociDuckClient {++ private final FlociDuckManager duckManager;+ private final EmulatorConfig config;+ private final EmbeddedDnsServer embeddedDnsServer;+ private final DockerHostResolver dockerHostResolver;+ private final ObjectMapper mapper;+ private final HttpClient httpClient;++ @Inject+ public FlociDuckClient(FlociDuckManager duckManager,+ EmulatorConfig config,+ EmbeddedDnsServer embeddedDnsServer,+ DockerHostResolver dockerHostResolver,+ ObjectMapper mapper) {+ this.duckManager = duckManager;+ this.config = config;+ this.embeddedDnsServer = embeddedDnsServer;+ this.dockerHostResolver = dockerHostResolver;+ this.mapper = mapper;+ this.httpClient = HttpClient.newHttpClient();+ }++ /**+ * Returns true if floci-duck is currently reachable without starting it.+ */+ public boolean isAvailable() {+ return duckManager.isAvailable();+ }++ /**+ * Executes SQL via floci-duck and writes the result as a CSV to {@code outputS3Path}.+ * Used by Athena query execution.+ *+ * @param sql the SQL to run (FROM clauses reference s3:// paths)+ * @param setupDdl optional DDL to run before the query (e.g. CREATE VIEW)+ * @param outputS3Path the S3 path where the result CSV will be written+ */+ public void execute(String sql, String setupDdl, String outputS3Path) {+ String duckUrl = duckManager.ensureReady();++ Map<String, Object> body = buildBaseBody(sql, setupDdl);+ body.put("output_s3_path", outputS3Path);++ post(duckUrl + "/execute", body, "execute");+ }++ /**+ * Executes SQL via floci-duck and returns the result rows as a list of column→value maps.+ * Used by S3 Select for formats that require DuckDB evaluation (e.g. Parquet).+ *+ * @param sql the SQL to run (FROM clauses reference s3:// paths)+ * @param setupDdl optional DDL to run before the query+ * @return result rows; empty list if the query matched no rows+ */+ public List<Map<String, Object>> query(String sql, String setupDdl) {+ String duckUrl = duckManager.ensureReady();++ Map<String, Object> body = buildBaseBody(sql, setupDdl);++ String responseBody = post(duckUrl + "/query", body, "query");+ return parseQueryRows(responseBody);+ }++ // ── internals ─────────────────────────────────────────────────────────────++ private Map<String, Object> buildBaseBody(String sql, String setupDdl) {+ Map<String, Object> body = new LinkedHashMap<>();+ body.put("sql", sql);+ if (setupDdl != null && !setupDdl.isBlank()) {+ body.put("setup_sql", setupDdl);+ }+ body.put("s3_endpoint", resolveFlociEndpoint());+ body.put("s3_region", config.defaultRegion());+ body.put("s3_access_key", "test");+ body.put("s3_secret_key", "test");+ body.put("s3_url_style", "path");+ return body;+ }++ /**+ * Returns the Floci S3 endpoint URL as reachable from inside a floci-duck container.+ * Uses the embedded DNS hostname when the DNS server is active, falls back to the+ * raw Docker host IP for local dev mode.+ */+ private String resolveFlociEndpoint() {+ int port = URI.create(config.baseUrl()).getPort();+ String hostname = embeddedDnsServer.getServerIp().isPresent()+ ? config.hostname().orElse(EmbeddedDnsServer.DEFAULT_SUFFIX)+ : dockerHostResolver.resolve();+ return "http://" + hostname + ":" + port;+ }++ private String post(String url, Map<String, Object> body, String operation) {+ try {+ String json = mapper.writeValueAsString(body);+ HttpRequest request = HttpRequest.newBuilder()+ .uri(URI.create(url))+ .header("Content-Type", "application/json")+ .POST(HttpRequest.BodyPublishers.ofString(json))+ .build();+ HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());+ if (response.statusCode() != 200) {+ throw new RuntimeException("floci-duck " + operation + " returned HTTP "+ + response.statusCode() + ": " + response.body());+ }+ return response.body();+ } catch (RuntimeException e) {+ throw e;+ } catch (Exception e) {+ throw new RuntimeException("Failed to call floci-duck " + operation + ": " + e.getMessage(), e);+ }+ }++ @SuppressWarnings("unchecked")+ private List<Map<String, Object>> parseQueryRows(String responseBody) {+ try {+ JsonNode root = mapper.readTree(responseBody);+ if (!"success".equals(root.path("status").asText())) {+ throw new RuntimeException("floci-duck query error: " + root.path("message").asText());+ }+ JsonNode rowsNode = root.path("rows");+ if (rowsNode.isMissingNode() || rowsNode.isNull()) {+ return List.of();+ }+ List<Map<String, Object>> rows = new ArrayList<>();+ for (JsonNode row : rowsNode) {+ rows.add((Map<String, Object>) mapper.treeToValue(row, Map.class));+ }+ return rows;+ } catch (RuntimeException e) {+ throw e;+ } catch (Exception e) {+ throw new RuntimeException("Failed to parse floci-duck query response: " + e.getMessage(), e);+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/FlociDuckManager.java b/src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckManager.javasimilarity index 79%rename from src/main/java/io/github/hectorvent/floci/services/athena/FlociDuckManager.javarename to src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckManager.javaindex a2d3834f17..8c93aa2bc6 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/FlociDuckManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckManager.java@@ -1,4 +1,4 @@-package io.github.hectorvent.floci.services.athena;+package io.github.hectorvent.floci.services.floci;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;@@ -23,7 +23,7 @@ import java.util.Optional;* On the first call to {@link #ensureReady()}, Floci pulls the image and starts* a named container "floci-duck". Subsequent calls return the cached URL immediately.*- * If {@code floci.services.athena.duck-url} is configured, container management is+ * If {@code floci.services.duck.url} is configured, container management is* skipped entirely and that URL is used as-is — useful in Docker Compose setups where* the user runs floci-duck as a separate service.*/@@ -55,6 +55,37 @@ public class FlociDuckManager {this.config = config;}+ /**+ * Returns true if floci-duck is reachable without starting anything.+ * If a URL is pre-configured and healthy, that URL is cached for future ensureReady() calls.+ */+ public synchronized boolean isAvailable() {+ if (resolvedUrl != null) {+ return true;+ }+ Optional<String> configured = config.services().duck().url();+ if (configured.isPresent() && !configured.get().isBlank()) {+ String url = configured.get();+ if (probeHealth(url)) {+ resolvedUrl = url;+ LOG.infov("floci-duck is available at pre-configured URL: {0}", url);+ return true;+ }+ }+ return false;+ }++ private boolean probeHealth(String baseUrl) {+ try {+ HttpURLConnection conn = (HttpURLConnection) URI.create(baseUrl + "/health").toURL().openConnection();+ conn.setConnectTimeout(500);+ conn.setReadTimeout(500);+ return conn.getResponseCode() == 200;+ } catch (Exception e) {+ return false;+ }+ }+/*** Returns the floci-duck base URL, starting the container on first call if needed.* Thread-safe — concurrent callers wait while the first thread does the pull+start.@@ -64,7 +95,7 @@ public class FlociDuckManager {return resolvedUrl;}- Optional<String> configured = config.services().athena().duckUrl();+ Optional<String> configured = config.services().duck().url();if (configured.isPresent() && !configured.get().isBlank()) {resolvedUrl = configured.get();LOG.infov("Using pre-configured floci-duck URL: {0}", resolvedUrl);@@ -76,7 +107,7 @@ public class FlociDuckManager {}private void startContainer() {- String image = config.services().athena().defaultImage();+ String image = config.services().duck().defaultImage();LOG.infov("Starting floci-duck container using image {0}", image);lifecycleManager.removeIfExists(CONTAINER_NAME);diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.javaindex 9bfa4bd671..4512eb0ee7 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.java@@ -1,14 +1,18 @@package io.github.hectorvent.floci.services.s3;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.List;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.CsvParser;+import org.jboss.logging.Logger;++import java.util.*;import java.util.regex.Matcher;import java.util.regex.Pattern;import java.util.stream.Collectors;-import org.jboss.logging.Logger;+import java.util.stream.StreamSupport;-public class S3SelectEvaluator {+class S3SelectEvaluator {private static final Logger LOG = Logger.getLogger(S3SelectEvaluator.class);@@ -16,13 +20,195 @@ public class S3SelectEvaluator {"SELECT\\s+(.+?)\\s+FROM\\s+S3OBJECT(?:\\s+(\\w+))?\\s*(?:WHERE\\s+(.+?))?\\s*(?:LIMIT\\s+(\\d+))?\\s*$",Pattern.CASE_INSENSITIVE | Pattern.DOTALL);- public static String evaluateCsv(String content, String expression, boolean useHeaders) {+ // ── AST node types ─────────────────────────────────────────────────────++ private sealed interface WhereNode+ permits AndNode, OrNode, NotNode, CompNode, IsNullNode, BetweenNode, InNode, LikeNode {}++ private record AndNode(WhereNode left, WhereNode right) implements WhereNode {}+ private record OrNode(WhereNode left, WhereNode right) implements WhereNode {}+ private record NotNode(WhereNode child) implements WhereNode {}+ private record CompNode(String col, String op, String val) implements WhereNode {}+ private record IsNullNode(String col, boolean negate) implements WhereNode {}+ private record BetweenNode(String col, String lo, String hi) implements WhereNode {}+ private record InNode(String col, List<String> vals) implements WhereNode {}+ private record LikeNode(String col, String pattern) implements WhereNode {}++ // ── Lexer ──────────────────────────────────────────────────────────────++ private enum TokenType { IDENT, STRING, NUMBER, OP, KEYWORD, LPAREN, RPAREN, COMMA }++ private record Token(TokenType type, String value) {}++ private static final Set<String> KEYWORDS = Set.of(+ "AND", "OR", "NOT", "LIKE", "BETWEEN", "IN", "IS", "NULL", "TRUE", "FALSE"+ );++ private static List<Token> tokenize(String input) {+ List<Token> tokens = new ArrayList<>();+ int i = 0;+ while (i < input.length()) {+ char c = input.charAt(i);+ if (Character.isWhitespace(c)) { i++; continue; }+ if (c == '(') { tokens.add(new Token(TokenType.LPAREN, "(")); i++; continue; }+ if (c == ')') { tokens.add(new Token(TokenType.RPAREN, ")")); i++; continue; }+ if (c == ',') { tokens.add(new Token(TokenType.COMMA, ",")); i++; continue; }++ if (c == '\'') {+ StringBuilder sb = new StringBuilder();+ i++;+ while (i < input.length()) {+ char ch = input.charAt(i);+ if (ch == '\'' && i + 1 < input.length() && input.charAt(i + 1) == '\'') {+ sb.append('\''); i += 2;+ } else if (ch == '\'') {+ i++; break;+ } else {+ sb.append(ch); i++;+ }+ }+ tokens.add(new Token(TokenType.STRING, sb.toString()));+ continue;+ }++ if (Character.isDigit(c) || (c == '-' && i + 1 < input.length() && Character.isDigit(input.charAt(i + 1)))) {+ int start = i++;+ while (i < input.length() && (Character.isDigit(input.charAt(i)) || input.charAt(i) == '.')) i++;+ tokens.add(new Token(TokenType.NUMBER, input.substring(start, i)));+ continue;+ }++ if (i + 1 < input.length()) {+ String two = input.substring(i, i + 2);+ if (">=".equals(two) || "<=".equals(two) || "<>".equals(two) || "!=".equals(two)) {+ tokens.add(new Token(TokenType.OP, two)); i += 2; continue;+ }+ }+ if (c == '>' || c == '<' || c == '=') {+ tokens.add(new Token(TokenType.OP, String.valueOf(c))); i++; continue;+ }++ if (Character.isLetter(c) || c == '_') {+ int start = i++;+ while (i < input.length() && (Character.isLetterOrDigit(input.charAt(i))+ || input.charAt(i) == '_' || input.charAt(i) == '.')) i++;+ String word = input.substring(start, i);+ String upper = word.toUpperCase();+ if (KEYWORDS.contains(upper)) {+ tokens.add(new Token(TokenType.KEYWORD, upper));+ } else {+ tokens.add(new Token(TokenType.IDENT, word));+ }+ continue;+ }+ i++;+ }+ return tokens;+ }++ // ── Recursive descent parser ───────────────────────────────────────────++ private static final class Parser {+ private final List<Token> tokens;+ private int pos;++ Parser(List<Token> tokens) { this.tokens = tokens; }++ WhereNode parse() { return orExpr(); }++ private WhereNode orExpr() {+ WhereNode left = andExpr();+ while (peekKeyword("OR")) { advance(); left = new OrNode(left, andExpr()); }+ return left;+ }++ private WhereNode andExpr() {+ WhereNode left = notExpr();+ while (peekKeyword("AND")) { advance(); left = new AndNode(left, notExpr()); }+ return left;+ }++ private WhereNode notExpr() {+ if (peekKeyword("NOT")) { advance(); return new NotNode(notExpr()); }+ return atom();+ }++ private WhereNode atom() {+ if (pos < tokens.size() && tokens.get(pos).type() == TokenType.LPAREN) {+ advance();+ WhereNode inner = orExpr();+ if (pos < tokens.size() && tokens.get(pos).type() == TokenType.RPAREN) advance();+ return inner;+ }+ String col = consumeOperand();+ if (pos >= tokens.size()) return new CompNode(col, "=", "");+ Token next = tokens.get(pos);+ if (next.type() == TokenType.KEYWORD) {+ return switch (next.value()) {+ case "IS" -> {+ advance();+ boolean negate = peekKeyword("NOT");+ if (negate) advance();+ if (peekKeyword("NULL")) advance();+ yield new IsNullNode(col, negate);+ }+ case "BETWEEN" -> {+ advance();+ String lo = consumeOperand();+ if (peekKeyword("AND")) advance();+ yield new BetweenNode(col, lo, consumeOperand());+ }+ case "IN" -> {+ advance();+ if (pos < tokens.size() && tokens.get(pos).type() == TokenType.LPAREN) advance();+ List<String> vals = new ArrayList<>();+ while (pos < tokens.size() && tokens.get(pos).type() != TokenType.RPAREN) {+ vals.add(consumeOperand());+ if (pos < tokens.size() && tokens.get(pos).type() == TokenType.COMMA) advance();+ }+ if (pos < tokens.size() && tokens.get(pos).type() == TokenType.RPAREN) advance();+ yield new InNode(col, vals);+ }+ case "LIKE" -> {+ advance();+ yield new LikeNode(col, consumeOperand());+ }+ default -> new CompNode(col, "=", "");+ };+ }+ if (next.type() == TokenType.OP) {+ String op = next.value(); advance();+ return new CompNode(col, op, consumeOperand());+ }+ return new CompNode(col, "=", "");+ }++ private String consumeOperand() {+ if (pos >= tokens.size()) return "";+ return tokens.get(pos++).value();+ }++ private boolean peekKeyword(String kw) {+ return pos < tokens.size()+ && tokens.get(pos).type() == TokenType.KEYWORD+ && kw.equals(tokens.get(pos).value());+ }++ private void advance() { if (pos < tokens.size()) pos++; }+ }++ private static WhereNode parseWhere(String where) {+ return new Parser(tokenize(where)).parse();+ }++ // ── CSV evaluation ─────────────────────────────────────────────────────++ static String evaluateCsv(String content, String expression, String fileHeaderInfo, String outputFormat) {Matcher matcher = SELECT_PATTERN.matcher(expression.trim());if (!matcher.find()) {- LOG.debugv("SQL Pattern did not match: {0}", expression);+ LOG.debugv("SQL pattern did not match: {0}", expression);return content;}-String projection = matcher.group(1).trim();String alias = matcher.group(2);String whereClause = matcher.group(3);@@ -33,104 +219,312 @@ public class S3SelectEvaluator {List<String> headerList = new ArrayList<>();int dataStart = 0;- if (useHeaders) {- headerList = Arrays.asList(lines[0].split(","));+ if ("USE".equalsIgnoreCase(fileHeaderInfo)) {+ headerList = CsvParser.parseLine(lines[0]);+ dataStart = 1;+ } else if ("IGNORE".equalsIgnoreCase(fileHeaderInfo)) {dataStart = 1;}List<String[]> rows = new ArrayList<>();for (int i = dataStart; i < lines.length; i++) {if (lines[i].trim().isEmpty()) continue;- rows.add(lines[i].split(","));+ rows.add(CsvParser.parseLine(lines[i]).toArray(String[]::new));}- // Filterif (whereClause != null) {- rows = filterRows(rows, headerList, alias, whereClause);+ rows = filterCsvRows(rows, headerList, alias, whereClause);}- // Limitif (limitStr != null) {int limit = Integer.parseInt(limitStr);- if (rows.size() > limit) {- rows = rows.subList(0, limit);- }+ if (rows.size() > limit) rows = rows.subList(0, limit);}- // Project- return projectRows(rows, headerList, projection);+ return projectCsvRows(rows, headerList, projection, outputFormat);}- private static List<String[]> filterRows(List<String[]> rows, List<String> headers, String alias, String where) {- String processedWhere = where;+ private static List<String[]> filterCsvRows(List<String[]> rows, List<String> headers,+ String alias, String where) {+ String processed = where;if (alias != null) {- processedWhere = processedWhere.replaceAll("(?i)" + Pattern.quote(alias) + "\\.", "");- }- final String finalWhere = processedWhere;-- return rows.stream().filter(row -> {- // Simple evaluation: col > val or col = val- // Try to find a comparison- Pattern compPattern = Pattern.compile("(\\w+)\\s*(>|=|<)\\s*(\\d+|'[^']*')", Pattern.CASE_INSENSITIVE);- Matcher m = compPattern.matcher(finalWhere);- if (m.find()) {- String colName = m.group(1);- String op = m.group(2);- String valStr = m.group(3);-- String cellValue = getCellValue(row, headers, colName);- if (cellValue == null) return false;-- if (valStr.startsWith("'")) {- String val = valStr.substring(1, valStr.length() - 1);- return op.equals("=") && cellValue.equalsIgnoreCase(val);- } else {- try {- double cellNum = Double.parseDouble(cellValue);- double valNum = Double.parseDouble(valStr);- return switch (op) {- case ">" -> cellNum > valNum;- case "<" -> cellNum < valNum;- case "=" -> cellNum == valNum;- default -> false;- };- } catch (NumberFormatException e) {- return false;- }- }+ processed = processed.replaceAll("(?i)" + Pattern.quote(alias) + "\\.", "");+ }+ try {+ WhereNode tree = parseWhere(processed);+ return rows.stream().filter(row -> evalCsv(tree, row, headers)).toList();+ } catch (Exception e) {+ LOG.warnv("WHERE parse error, returning all rows: {0}", e.getMessage());+ return rows;+ }+ }++ private static boolean evalCsv(WhereNode node, String[] row, List<String> headers) {+ return switch (node) {+ case AndNode(var l, var r) -> evalCsv(l, row, headers) && evalCsv(r, row, headers);+ case OrNode(var l, var r) -> evalCsv(l, row, headers) || evalCsv(r, row, headers);+ case NotNode(var child) -> !evalCsv(child, row, headers);+ case IsNullNode(var col, var negate) -> {+ String v = getCellValue(row, headers, col);+ yield negate ? v != null : v == null;+ }+ case BetweenNode(var col, var lo, var hi) -> {+ String v = getCellValue(row, headers, col);+ yield v != null && cmp(v, lo) >= 0 && cmp(v, hi) <= 0;+ }+ case InNode(var col, var vals) -> {+ String v = getCellValue(row, headers, col);+ yield v != null && vals.stream().anyMatch(v::equals);}- return true;- }).collect(Collectors.toList());+ case LikeNode(var col, var pattern) -> {+ String v = getCellValue(row, headers, col);+ yield v != null && matchLike(v, pattern);+ }+ case CompNode(var col, var op, var val) -> {+ String v = getCellValue(row, headers, col);+ yield v != null && compareOp(v, op, val);+ }+ };}- private static String projectRows(List<String[]> rows, List<String> headers, String projection) {- if (projection.equals("*")) {+ private static String projectCsvRows(List<String[]> rows, List<String> headers,+ String projection, String outputFormat) {+ boolean toJson = "JSON".equalsIgnoreCase(outputFormat);+ if ("*".equals(projection)) {return rows.stream()- .map(row -> String.join(",", row))+ .map(row -> toJson ? rowToJson(row, headers) : String.join(",", row)).collect(Collectors.joining("\n")) + (rows.isEmpty() ? "" : "\n");}-- String[] cols = projection.split(",");+ String[] cols = Arrays.stream(projection.split(",")).map(String::trim).toArray(String[]::new);return rows.stream().map(row -> {- List<String> projected = new ArrayList<>();- for (String col : cols) {- projected.add(getCellValue(row, headers, col.trim()));+ List<String> vals = Arrays.stream(cols)+ .map(col -> { String v = getCellValue(row, headers, col); return v != null ? v : ""; })+ .toList();+ if (toJson) {+ StringBuilder json = new StringBuilder("{");+ for (int i = 0; i < cols.length; i++) {+ if (i > 0) json.append(",");+ json.append('"').append(jsonEscape(cols[i])).append("\":\"")+ .append(jsonEscape(vals.get(i))).append('"');+ }+ return json.append("}").toString();}- return String.join(",", projected);+ return String.join(",", vals);}).collect(Collectors.joining("\n")) + (rows.isEmpty() ? "" : "\n");}+ // ── Duck rows formatting ───────────────────────────────────────────────++ /**+ * Serializes rows returned by floci-duck (/query) into the requested output format.+ * DuckDB already applied filtering and projection, so this is pure serialization.+ */+ static String formatDuckRows(List<Map<String, Object>> rows, String outputFormat, ObjectMapper mapper) {+ boolean toJson = "JSON".equalsIgnoreCase(outputFormat);+ StringBuilder sb = new StringBuilder();+ for (Map<String, Object> row : rows) {+ if (toJson) {+ try {+ sb.append(mapper.writeValueAsString(row)).append("\n");+ } catch (Exception ignored) {}+ } else {+ sb.append(row.values().stream()+ .map(v -> v == null ? "" : csvEscape(v.toString()))+ .collect(Collectors.joining(","))).append("\n");+ }+ }+ return sb.toString();+ }++ // ── JSON evaluation ────────────────────────────────────────────────────++ static String evaluateJson(String content, String expression, ObjectMapper mapper, String outputFormat) {+ Matcher matcher = SELECT_PATTERN.matcher(expression.trim());+ if (!matcher.find()) {+ LOG.debugv("SQL pattern did not match for JSON: {0}", expression);+ return content;+ }+ String projection = matcher.group(1).trim();+ String alias = matcher.group(2);+ String whereClause = matcher.group(3);+ String limitStr = matcher.group(4);++ WhereNode tree = null;+ if (whereClause != null) {+ String processed = whereClause;+ if (alias != null) {+ processed = processed.replaceAll("(?i)" + Pattern.quote(alias) + "\\.", "");+ }+ try {+ tree = parseWhere(processed);+ } catch (Exception e) {+ LOG.warnv("JSON WHERE parse error: {0}", e.getMessage());+ }+ }++ List<JsonNode> rows = parseJsonInput(content, mapper);++ if (tree != null) {+ final WhereNode finalTree = tree;+ rows = rows.stream().filter(row -> evalJson(finalTree, row)).toList();+ }++ if (limitStr != null) {+ int limit = Integer.parseInt(limitStr);+ if (rows.size() > limit) rows = rows.subList(0, limit);+ }++ return projectJsonRows(rows, projection, outputFormat, mapper);+ }++ private static List<JsonNode> parseJsonInput(String content, ObjectMapper mapper) {+ // Try JSON array first (starts with '['), then JSON Lines, then single object.+ if (content.trim().startsWith("[")) {+ try {+ JsonNode root = mapper.readTree(content);+ if (root.isArray()) {+ return StreamSupport.stream(root.spliterator(), false).toList();+ }+ } catch (Exception ignored) {}+ }+ // JSON Lines: parse each non-empty line independently.+ List<JsonNode> rows = new ArrayList<>();+ for (String line : content.split("\\r?\\n")) {+ if (line.trim().isEmpty()) continue;+ try { rows.add(mapper.readTree(line)); } catch (Exception ignored) {}+ }+ return rows;+ }++ private static boolean evalJson(WhereNode node, JsonNode row) {+ return switch (node) {+ case AndNode(var l, var r) -> evalJson(l, row) && evalJson(r, row);+ case OrNode(var l, var r) -> evalJson(l, row) || evalJson(r, row);+ case NotNode(var child) -> !evalJson(child, row);+ case IsNullNode(var col, var negate) -> {+ JsonNode v = row.path(col);+ boolean isNull = v.isMissingNode() || v.isNull();+ yield negate ? !isNull : isNull;+ }+ case BetweenNode(var col, var lo, var hi) -> {+ String v = row.path(col).asText(null);+ yield v != null && cmp(v, lo) >= 0 && cmp(v, hi) <= 0;+ }+ case InNode(var col, var vals) -> {+ String v = row.path(col).asText(null);+ yield v != null && vals.stream().anyMatch(v::equals);+ }+ case LikeNode(var col, var pattern) -> {+ String v = row.path(col).asText(null);+ yield v != null && matchLike(v, pattern);+ }+ case CompNode(var col, var op, var val) -> {+ String v = row.path(col).asText(null);+ yield v != null && compareOp(v, op, val);+ }+ };+ }++ private static String projectJsonRows(List<JsonNode> rows, String projection,+ String outputFormat, ObjectMapper mapper) {+ boolean toCsv = "CSV".equalsIgnoreCase(outputFormat);+ String[] cols = "*".equals(projection) ? null+ : Arrays.stream(projection.split(",")).map(String::trim).toArray(String[]::new);+ StringBuilder sb = new StringBuilder();+ for (JsonNode row : rows) {+ if (cols == null) {+ if (toCsv) {+ List<String> vals = new ArrayList<>();+ row.fields().forEachRemaining(e -> vals.add(csvEscape(e.getValue().asText(""))));+ sb.append(String.join(",", vals)).append("\n");+ } else {+ try { sb.append(mapper.writeValueAsString(row)).append("\n"); } catch (Exception ignored) {}+ }+ } else {+ if (toCsv) {+ sb.append(Arrays.stream(cols)+ .map(col -> csvEscape(row.path(col).asText("")))+ .collect(Collectors.joining(","))).append("\n");+ } else {+ ObjectNode projected = mapper.createObjectNode();+ for (String col : cols) {+ JsonNode v = row.path(col);+ if (!v.isMissingNode()) projected.set(col, v);+ }+ try { sb.append(mapper.writeValueAsString(projected)).append("\n"); } catch (Exception ignored) {}+ }+ }+ }+ return sb.toString();+ }++ // ── Shared helpers ─────────────────────────────────────────────────────++ private static boolean compareOp(String cell, String op, String val) {+ int c = cmp(cell, val);+ return switch (op) {+ case "=" -> c == 0;+ case "<>", "!=" -> c != 0;+ case ">" -> c > 0;+ case "<" -> c < 0;+ case ">=" -> c >= 0;+ case "<=" -> c <= 0;+ default -> false;+ };+ }++ private static int cmp(String a, String b) {+ try {+ return Double.compare(Double.parseDouble(a), Double.parseDouble(b));+ } catch (NumberFormatException e) {+ return a.compareTo(b);+ }+ }++ private static boolean matchLike(String value, String pattern) {+ StringBuilder regex = new StringBuilder("^");+ for (int i = 0; i < pattern.length(); i++) {+ char c = pattern.charAt(i);+ if (c == '%') regex.append(".*");+ else if (c == '_') regex.append(".");+ else regex.append(Pattern.quote(String.valueOf(c)));+ }+ return Pattern.compile(regex.append("$").toString(), Pattern.DOTALL).matcher(value).matches();+ }++ private static String rowToJson(String[] row, List<String> headers) {+ StringBuilder json = new StringBuilder("{");+ for (int i = 0; i < headers.size(); i++) {+ if (i > 0) json.append(",");+ String val = i < row.length ? row[i] : "";+ json.append('"').append(jsonEscape(headers.get(i).trim())).append("\":\"")+ .append(jsonEscape(val)).append('"');+ }+ return json.append("}").toString();+ }++ private static String jsonEscape(String s) {+ return s.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n").replace("\r", "\\r");+ }++ private static String csvEscape(String s) {+ if (s.contains(",") || s.contains("\"") || s.contains("\n")) {+ return "\"" + s.replace("\"", "\"\"") + "\"";+ }+ return s;+ }+private static String getCellValue(String[] row, List<String> headers, String colName) {if (colName.startsWith("_")) {try {int idx = Integer.parseInt(colName.substring(1)) - 1;- return idx < row.length ? row[idx] : null;+ return idx >= 0 && idx < row.length ? row[idx] : null;} catch (NumberFormatException e) {return null;}}for (int i = 0; i < headers.size(); i++) {- if (headers.get(i).equalsIgnoreCase(colName)) {+ if (headers.get(i).trim().equalsIgnoreCase(colName)) {return i < row.length ? row[i] : null;}}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.javaindex 6f9f7ad541..50144b011b 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.java@@ -2,62 +2,123 @@ package io.github.hectorvent.floci.services.s3;import io.github.hectorvent.floci.core.common.AwsEventStreamEncoder;import io.github.hectorvent.floci.core.common.XmlParser;+import io.github.hectorvent.floci.services.floci.FlociDuckClient;import io.github.hectorvent.floci.services.s3.model.S3Object;-import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import java.io.ByteArrayOutputStream;import java.nio.charset.StandardCharsets;import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.regex.Matcher;+import java.util.regex.Pattern;@ApplicationScopedpublic class S3SelectService {+ private static final Pattern S3OBJECT_PATTERN =+ Pattern.compile("\\bS3Object\\b", Pattern.CASE_INSENSITIVE);+private final ObjectMapper objectMapper;+ private final FlociDuckClient duckClient;@Inject- public S3SelectService(ObjectMapper objectMapper) {+ public S3SelectService(ObjectMapper objectMapper, FlociDuckClient duckClient) {this.objectMapper = objectMapper;+ this.duckClient = duckClient;}public byte[] select(S3Object object, String requestXml) {- String expression = XmlParser.extractFirst(requestXml, "Expression", "").toUpperCase();- String inputType = requestXml.contains("<CSV>") ? "CSV" : (requestXml.contains("<JSON>") ? "JSON" : null);-+ String expression = XmlParser.extractFirst(requestXml, "Expression", "");+ String fileHeaderInfo = XmlParser.extractFirst(requestXml, "FileHeaderInfo", "NONE");+ String inputType = detectType(requestXml, "InputSerialization");+ String outputFormat = detectType(requestXml, "OutputSerialization");+ if (outputFormat == null) outputFormat = "CSV";+byte[] rawData = object.getData();if (rawData == null) return new byte[0];- String content = new String(rawData, StandardCharsets.UTF_8);-- StringBuilder result = new StringBuilder();-- if ("CSV".equals(inputType)) {- boolean useHeaders = requestXml.contains("<FileHeaderInfo>USE</FileHeaderInfo>");- String filtered = S3SelectEvaluator.evaluateCsv(content, expression, useHeaders);- result.append(filtered);+ long bytesScanned = rawData.length;++ String result;+ if (isParquet(object)) {+ result = selectParquet(object, expression, outputFormat);+ } else if (duckClient.isAvailable() && canUseDuck(inputType, fileHeaderInfo)) {+ result = selectViaDuck(object, expression, inputType, outputFormat);+ } else if ("CSV".equals(inputType)) {+ String content = new String(rawData, StandardCharsets.UTF_8);+ result = S3SelectEvaluator.evaluateCsv(content, expression, fileHeaderInfo, outputFormat);} else if ("JSON".equals(inputType)) {- // Assume one JSON object per line (JSON Lines) or a single array- try {- JsonNode node = objectMapper.readTree(content);- if (node.isArray()) {- for (JsonNode item : node) {- result.append(objectMapper.writeValueAsString(item)).append("\n");- }- } else {- result.append(objectMapper.writeValueAsString(node)).append("\n");- }- } catch (Exception e) {- // If it's not valid JSON, just return raw or fail- result.append(content);- }+ String content = new String(rawData, StandardCharsets.UTF_8);+ result = S3SelectEvaluator.evaluateJson(content, expression, objectMapper, outputFormat);} else {- result.append(content);+ result = new String(rawData, StandardCharsets.UTF_8);}- return encodeEventStream(result.toString());+ long bytesReturned = result.getBytes(StandardCharsets.UTF_8).length;+ return encodeEventStream(result, bytesScanned, bytesReturned);+ }++ // ── Duck delegation ────────────────────────────────────────────────────++ /**+ * Duck handles CSV with USE headers (named columns) and all JSON.+ * CSV NONE/IGNORE use _N positional refs that DuckDB doesn't natively support,+ * so those fall back to the Java evaluator.+ */+ private static boolean canUseDuck(String inputType, String fileHeaderInfo) {+ if ("JSON".equals(inputType)) {+ return true;+ }+ return "CSV".equals(inputType) && "USE".equals(fileHeaderInfo);+ }++ private String selectViaDuck(S3Object object, String expression, String inputType,+ String outputFormat) {+ String s3Uri = "s3://" + object.getBucketName() + "/" + object.getKey();+ String readFn = "JSON".equals(inputType)+ ? "read_json_auto('" + s3Uri + "')"+ : "read_csv('" + s3Uri + "', header=true, null_padding=true)";+ String duckSql = S3OBJECT_PATTERN.matcher(expression)+ .replaceAll(Matcher.quoteReplacement(readFn));+ List<Map<String, Object>> rows = duckClient.query(duckSql, null);+ return S3SelectEvaluator.formatDuckRows(rows, outputFormat, objectMapper);+ }++ // ── Parquet delegation ─────────────────────────────────────────────────++ private static boolean isParquet(S3Object object) {+ String ct = object.getContentType();+ if (ct != null && ct.toLowerCase().contains("parquet")) return true;+ String key = object.getKey();+ return key != null && key.toLowerCase().endsWith(".parquet");+ }++ private String selectParquet(S3Object object, String expression, String outputFormat) {+ String s3Uri = "s3://" + object.getBucketName() + "/" + object.getKey();+ String duckSql = S3OBJECT_PATTERN.matcher(expression)+ .replaceAll(Matcher.quoteReplacement("read_parquet('" + s3Uri + "')"));++ List<Map<String, Object>> rows = duckClient.query(duckSql, null);+ return S3SelectEvaluator.formatDuckRows(rows, outputFormat, objectMapper);+ }++ // ── Helpers ────────────────────────────────────────────────────────────++ private static String detectType(String requestXml, String sectionTag) {+ String openTag = "<" + sectionTag + ">";+ int start = requestXml.indexOf(openTag);+ if (start < 0) return null;+ int end = requestXml.indexOf("</" + sectionTag + ">", start);+ String section = end > start ? requestXml.substring(start, end) : requestXml.substring(start);+ if (section.contains("<CSV")) return "CSV";+ if (section.contains("<JSON")) return "JSON";+ if (section.contains("<Parquet")) return "PARQUET";+ return null;}- private byte[] encodeEventStream(String payload) {+ private byte[] encodeEventStream(String payload, long bytesScanned, long bytesReturned) {try {ByteArrayOutputStream baos = new ByteArrayOutputStream();@@ -71,8 +132,10 @@ public class S3SelectService {statsHeaders.put(":message-type", "event");statsHeaders.put(":event-type", "Stats");statsHeaders.put(":content-type", "text/xml");- byte[] statsPayload = "<Stats><BytesScanned>100</BytesScanned><BytesProcessed>100</BytesProcessed><BytesReturned>100</BytesReturned></Stats>".getBytes(StandardCharsets.UTF_8);- baos.write(AwsEventStreamEncoder.encodeMessage(statsHeaders, statsPayload));+ String statsXml = "<Stats><BytesScanned>" + bytesScanned + "</BytesScanned>"+ + "<BytesProcessed>" + bytesScanned + "</BytesProcessed>"+ + "<BytesReturned>" + bytesReturned + "</BytesReturned></Stats>";+ baos.write(AwsEventStreamEncoder.encodeMessage(statsHeaders, statsXml.getBytes(StandardCharsets.UTF_8)));LinkedHashMap<String, String> endHeaders = new LinkedHashMap<>();endHeaders.put(":message-type", "event");diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 432c385fd7..b621b92bc1 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -187,7 +187,8 @@ floci:athena:enabled: truemock: false- # duck-url: http://floci-duck:3000 # set to skip container management+ duck:+ # url: http://floci-duck:3000 # set to skip container managementdefault-image: "floci/floci-duck:latest"glue:enabled: truediff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.javaindex 6b40a1d70d..6a346a220a 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.java@@ -3,8 +3,6 @@ package io.github.hectorvent.floci.services.s3;import io.quarkus.test.junit.QuarkusTest;import org.junit.jupiter.api.Test;-import java.util.Base64;-import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.not;@@ -12,124 +10,311 @@ import static org.hamcrest.Matchers.not;@QuarkusTestclass S3SelectIntegrationTest {+ // ── helpers ───────────────────────────────────────────────────────────++ private static void createBucketAndPut(String bucket, String key, String body) {+ given().header("Host", bucket + ".localhost").when().put("/").then().statusCode(200);+ given().header("Host", bucket + ".localhost").body(body).when().put("/" + key).then().statusCode(200);+ }++ private static io.restassured.response.ValidatableResponse select(+ String bucket, String key, String expression,+ String inputSerialization, String outputSerialization) {+ // XML-escape the expression so operators like < and & are valid inside <Expression>+ String safeExpr = expression.replace("&", "&").replace("<", "<");+ String requestXml = """+ <?xml version="1.0" encoding="UTF-8"?>+ <SelectObjectContentRequest>+ <Expression>%s</Expression>+ <ExpressionType>SQL</ExpressionType>+ <InputSerialization>%s</InputSerialization>+ <OutputSerialization>%s</OutputSerialization>+ </SelectObjectContentRequest>+ """.formatted(safeExpr, inputSerialization, outputSerialization);+ return given()+ .header("Host", bucket + ".localhost")+ .queryParam("select", "")+ .queryParam("select-type", "2")+ .body(requestXml)+ .when().post("/" + key)+ .then().statusCode(200);+ }++ private static final String CSV_OUT = "<CSV/>";+ private static final String JSON_OUT = "<JSON/>";+ private static final String CSV_USE = "<CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV>";+ private static final String CSV_NONE = "<CSV><FileHeaderInfo>NONE</FileHeaderInfo></CSV>";+ private static final String CSV_IGNORE = "<CSV><FileHeaderInfo>IGNORE</FileHeaderInfo></CSV>";+ private static final String JSON_IN = "<JSON/>";++ // ── Existing tests (preserved) ────────────────────────────────────────+@Testvoid select_withWhereClause() {String bucket = "select-bucket";- String key = "data.csv";- String csvData = "name,age\nAlice,30\nBob,25\nCharlie,35";-- // 1. Create bucket- given()- .header("Host", bucket + ".localhost")- .when()- .put("/")- .then()- .statusCode(200);-- // 2. Put object- given()- .header("Host", bucket + ".localhost")- .body(csvData)- .when()- .put("/" + key)- .then()- .statusCode(200);-- // 3. Select with WHERE clause- String requestXml = """- <?xml version="1.0" encoding="UTF-8"?>- <SelectObjectContentRequest>- <Expression>SELECT * FROM S3Object s WHERE s.age > 30</Expression>- <ExpressionType>SQL</ExpressionType>- <InputSerialization>- <CSV>- <FileHeaderInfo>USE</FileHeaderInfo>- </CSV>- </InputSerialization>- <OutputSerialization>- <CSV/>- </OutputSerialization>- </SelectObjectContentRequest>- """;-- given()- .header("Host", bucket + ".localhost")- .queryParam("select", "")- .queryParam("select-type", "2")- .body(requestXml)- .when()- .post("/" + key)- .then()- .statusCode(200)- .body(containsString("Charlie,35"))- .body(not(containsString("Alice,30")))- .body(not(containsString("Bob,25")));+ createBucketAndPut(bucket, "data.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ select(bucket, "data.csv", "SELECT * FROM S3Object s WHERE s.age > 30", CSV_USE, CSV_OUT)+ .body(containsString("Charlie,35"))+ .body(not(containsString("Alice,30")))+ .body(not(containsString("Bob,25")));}@Testvoid select_withProjection() {String bucket = "select-bucket-proj";- String key = "data.csv";- String csvData = "name,age,city\nAlice,30,New York\nBob,25,London";-- given().header("Host", bucket + ".localhost").when().put("/").then().statusCode(200);- given().header("Host", bucket + ".localhost").body(csvData).when().put("/" + key).then().statusCode(200);-- String requestXml = """- <?xml version="1.0" encoding="UTF-8"?>- <SelectObjectContentRequest>- <Expression>SELECT name, city FROM S3Object</Expression>- <ExpressionType>SQL</ExpressionType>- <InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>- <OutputSerialization><CSV/></OutputSerialization>- </SelectObjectContentRequest>- """;-- given()- .header("Host", bucket + ".localhost")- .queryParam("select", "")- .queryParam("select-type", "2")- .body(requestXml)- .when()- .post("/" + key)- .then()- .statusCode(200)- .body(containsString("Alice,New York"))- .body(containsString("Bob,London"))- .body(not(containsString("30")))- .body(not(containsString("25")));+ createBucketAndPut(bucket, "data.csv", "name,age,city\nAlice,30,New York\nBob,25,London");+ select(bucket, "data.csv", "SELECT name, city FROM S3Object", CSV_USE, CSV_OUT)+ .body(containsString("Alice,New York"))+ .body(containsString("Bob,London"))+ .body(not(containsString("30")))+ .body(not(containsString("25")));}@Testvoid select_withLimit() {String bucket = "select-bucket-limit";- String key = "data.csv";- String csvData = "name,age\nAlice,30\nBob,25\nCharlie,35";+ createBucketAndPut(bucket, "data.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ select(bucket, "data.csv", "SELECT * FROM S3Object LIMIT 2", CSV_USE, CSV_OUT)+ .body(containsString("Alice,30"))+ .body(containsString("Bob,25"))+ .body(not(containsString("Charlie,35")));+ }- given().header("Host", bucket + ".localhost").when().put("/").then().statusCode(200);- given().header("Host", bucket + ".localhost").body(csvData).when().put("/" + key).then().statusCode(200);+ // ── A: String literal / toUpperCase fix ───────────────────────────────- String requestXml = """- <?xml version="1.0" encoding="UTF-8"?>- <SelectObjectContentRequest>- <Expression>SELECT * FROM S3Object LIMIT 2</Expression>- <ExpressionType>SQL</ExpressionType>- <InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>- <OutputSerialization><CSV/></OutputSerialization>- </SelectObjectContentRequest>- """;-- given()- .header("Host", bucket + ".localhost")- .queryParam("select", "")- .queryParam("select-type", "2")- .body(requestXml)- .when()- .post("/" + key)- .then()- .statusCode(200)- .body(containsString("Alice,30"))- .body(containsString("Bob,25"))- .body(not(containsString("Charlie,35")));+ @Test+ void select_whereStringEquality() {+ String bucket = "sel-str-eq";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE name = 'Alice'", CSV_USE, CSV_OUT)+ .body(containsString("Alice,30"))+ .body(not(containsString("Bob")))+ .body(not(containsString("Charlie")));+ }++ @Test+ void select_whereStringEquality_caseSensitive() {+ String bucket = "sel-str-case";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25");+ // 'alice' (lowercase) must NOT match 'Alice' — S3 Select = is case-sensitive+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE name = 'alice'", CSV_USE, CSV_OUT)+ .body(not(containsString("Alice")))+ .body(not(containsString("Bob")));+ }++ // ── B: FileHeaderInfo ─────────────────────────────────────────────────++ @Test+ void select_fileHeaderInfoNone() {+ String bucket = "sel-hdr-none";+ // With NONE, the first line is treated as data, positional refs must be used+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25");+ select(bucket, "d.csv", "SELECT _1 FROM S3Object", CSV_NONE, CSV_OUT)+ // header line itself appears as a data row+ .body(containsString("name"))+ .body(containsString("Alice"))+ .body(containsString("Bob"));+ }++ @Test+ void select_fileHeaderInfoIgnore() {+ String bucket = "sel-hdr-ignore";+ // With IGNORE, header row is skipped from output; positional refs work for data rows+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25");+ select(bucket, "d.csv", "SELECT _1 FROM S3Object", CSV_IGNORE, CSV_OUT)+ .body(containsString("Alice"))+ .body(containsString("Bob"))+ .body(not(containsString("name"))); // header row excluded+ }++ // ── C: Quoted CSV fields ──────────────────────────────────────────────++ @Test+ void select_quotedCsvField() {+ String bucket = "sel-quoted";+ // "Smith, John" contains a comma inside quotes — must not be split+ createBucketAndPut(bucket, "d.csv", "name,age\n\"Smith, John\",35\n\"Doe, Jane\",22");+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age > 30", CSV_USE, CSV_OUT)+ .body(containsString("Smith, John"))+ .body(not(containsString("Doe, Jane")));+ }++ // ── D: Enhanced WHERE operators ───────────────────────────────────────++ @Test+ void select_whereAnd() {+ String bucket = "sel-and";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,20\nCharlie,35");+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age > 20 AND age < 35", CSV_USE, CSV_OUT)+ .body(containsString("Alice,30"))+ .body(not(containsString("Bob")))+ .body(not(containsString("Charlie")));+ }++ @Test+ void select_whereOr() {+ String bucket = "sel-or";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE name = 'Alice' OR name = 'Charlie'", CSV_USE, CSV_OUT)+ .body(containsString("Alice"))+ .body(containsString("Charlie"))+ .body(not(containsString("Bob")));+ }++ @Test+ void select_whereNot() {+ String bucket = "sel-not";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE NOT age = 30", CSV_USE, CSV_OUT)+ .body(not(containsString("Alice")))+ .body(containsString("Bob,25"))+ .body(containsString("Charlie,35"));+ }++ @Test+ void select_whereLike() {+ String bucket = "sel-like";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nAlex,28");+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE name LIKE 'Al%'", CSV_USE, CSV_OUT)+ .body(containsString("Alice"))+ .body(containsString("Alex"))+ .body(not(containsString("Bob")));+ }++ @Test+ void select_whereIsNull() {+ String bucket = "sel-isnull";+ // Bob's row has no city field → city IS NULL+ createBucketAndPut(bucket, "d.csv", "name,age,city\nAlice,30,NY\nBob,25");+ select(bucket, "d.csv", "SELECT name FROM S3Object WHERE city IS NULL", CSV_USE, CSV_OUT)+ .body(containsString("Bob"))+ .body(not(containsString("Alice")));+ }++ @Test+ void select_whereBetween() {+ String bucket = "sel-between";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,20\nCharlie,25");+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age BETWEEN 25 AND 30", CSV_USE, CSV_OUT)+ .body(containsString("Alice,30"))+ .body(containsString("Charlie,25"))+ .body(not(containsString("Bob,20")));+ }++ @Test+ void select_whereIn() {+ String bucket = "sel-in";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE name IN ('Alice', 'Charlie')", CSV_USE, CSV_OUT)+ .body(containsString("Alice"))+ .body(containsString("Charlie"))+ .body(not(containsString("Bob")));+ }++ @Test+ void select_whereGreaterOrEqual() {+ String bucket = "sel-gte";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age >= 30", CSV_USE, CSV_OUT)+ .body(containsString("Alice,30")) // exactly 30 must be included+ .body(containsString("Charlie,35"))+ .body(not(containsString("Bob,25")));+ }++ @Test+ void select_whereLessOrEqual() {+ String bucket = "sel-lte";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age <= 25", CSV_USE, CSV_OUT)+ .body(containsString("Bob,25")) // exactly 25 must be included+ .body(not(containsString("Alice,30")))+ .body(not(containsString("Charlie,35")));+ }++ // ── E: JSON input evaluation ──────────────────────────────────────────++ @Test+ void select_jsonLinesWithWhere() {+ String bucket = "sel-json-where";+ String jsonLines = "{\"name\":\"Alice\",\"age\":30}\n{\"name\":\"Bob\",\"age\":25}\n{\"name\":\"Charlie\",\"age\":35}";+ createBucketAndPut(bucket, "d.json", jsonLines);+ select(bucket, "d.json", "SELECT * FROM S3Object WHERE age > 30", JSON_IN, JSON_OUT)+ .body(containsString("Charlie"))+ .body(not(containsString("Alice")))+ .body(not(containsString("Bob")));}++ @Test+ void select_jsonLinesWithProjection() {+ String bucket = "sel-json-proj";+ String jsonLines = "{\"name\":\"Alice\",\"age\":30,\"city\":\"NY\"}\n{\"name\":\"Bob\",\"age\":25,\"city\":\"LA\"}";+ createBucketAndPut(bucket, "d.json", jsonLines);+ select(bucket, "d.json", "SELECT name FROM S3Object", JSON_IN, JSON_OUT)+ .body(containsString("Alice"))+ .body(containsString("Bob"))+ // check JSON key absence rather than raw substring to avoid binary CRC false-positives+ .body(not(containsString("\"age\":")))+ .body(not(containsString("\"city\":")));+ }++ @Test+ void select_jsonArrayWithWhere() {+ String bucket = "sel-json-arr";+ String jsonArray = "[{\"name\":\"Alice\",\"age\":30},{\"name\":\"Bob\",\"age\":25}]";+ createBucketAndPut(bucket, "d.json", jsonArray);+ select(bucket, "d.json", "SELECT * FROM S3Object WHERE age < 30", JSON_IN, JSON_OUT)+ .body(containsString("Bob"))+ .body(not(containsString("Alice")));+ }++ @Test+ void select_jsonLinesIsNull() {+ String bucket = "sel-json-null";+ String jsonLines = "{\"name\":\"Alice\",\"age\":30,\"city\":\"NY\"}\n{\"name\":\"Bob\",\"age\":25}";+ createBucketAndPut(bucket, "d.json", jsonLines);+ select(bucket, "d.json", "SELECT name FROM S3Object WHERE city IS NULL", JSON_IN, JSON_OUT)+ .body(containsString("Bob"))+ .body(not(containsString("Alice")));+ }++ // ── F: Output format ──────────────────────────────────────────────────++ @Test+ void select_csvInputJsonOutput() {+ String bucket = "sel-csv-json-out";+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25");+ select(bucket, "d.csv", "SELECT * FROM S3Object", CSV_USE, JSON_OUT)+ .body(containsString("\"name\""))+ .body(containsString("\"Alice\""))+ .body(containsString("\"Bob\""));+ }++ @Test+ void select_jsonInputCsvOutput() {+ String bucket = "sel-json-csv-out";+ String jsonLines = "{\"name\":\"Alice\",\"age\":30}\n{\"name\":\"Bob\",\"age\":25}";+ createBucketAndPut(bucket, "d.json", jsonLines);+ select(bucket, "d.json", "SELECT name, age FROM S3Object", JSON_IN, CSV_OUT)+ .body(containsString("Alice,30"))+ .body(containsString("Bob,25"));+ }++ // ── G: Real stats bytes ───────────────────────────────────────────────++ @Test+ void select_statsReflectsActualBytes() {+ String bucket = "sel-stats";+ String csvData = "name,age\nAlice,30\nBob,25\nCharlie,35";+ createBucketAndPut(bucket, "d.csv", csvData);+ // WHERE filters out 2 of 3 rows, so BytesReturned < BytesScanned+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age > 30", CSV_USE, CSV_OUT)+ // Stats XML is embedded as UTF-8 in the binary event stream+ .body(containsString("<BytesScanned>"))+ .body(containsString("<BytesReturned>"))+ // Must not be the old hardcoded value+ .body(not(containsString("<BytesScanned>100</BytesScanned>")));+ }++ // Note: Parquet S3 Select tests (group H) require Docker + floci-duck and+ // are covered in compatibility-tests/ where the full stack is available.}