Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(s3): full SelectObjectContent evaluator with floci-duck integration (#835)

Hector Venturae725f38ac755
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3SelectTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3SelectTest.java
new file mode 100644
index 0000000000..c98cc819d6
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3SelectTest.java
@@ -0,0 +1,301 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.*;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.async.SdkPublisher;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import software.amazon.awssdk.http.Protocol;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.*;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.CompletableFuture;
+
+import static org.assertj.core.api.Assertions.*;
+
+@DisplayName("S3 SelectObjectContent")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class S3SelectTest {
+
+ private static S3Client s3;
+ private static S3AsyncClient s3Async;
+ private static final String BUCKET = "compat-select-bucket";
+
+ @BeforeAll
+ static void setup() {
+ s3 = TestFixtures.s3Client();
+ s3Async = S3AsyncClient.builder()
+ .endpointOverride(TestFixtures.endpoint())
+ .region(Region.US_EAST_1)
+ .credentialsProvider(StaticCredentialsProvider.create(
+ AwsBasicCredentials.create("test", "test")))
+ .forcePathStyle(true)
+ .httpClientBuilder(NettyNioAsyncHttpClient.builder().protocol(Protocol.HTTP1_1))
+ .build();
+ try {
+ s3.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build());
+ } catch (BucketAlreadyOwnedByYouException ignored) {}
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (s3 == null) return;
+ try {
+ ListObjectsV2Response list = s3.listObjectsV2(
+ ListObjectsV2Request.builder().bucket(BUCKET).build());
+ for (S3Object obj : list.contents()) {
+ s3.deleteObject(DeleteObjectRequest.builder()
+ .bucket(BUCKET).key(obj.key()).build());
+ }
+ s3.deleteBucket(DeleteBucketRequest.builder().bucket(BUCKET).build());
+ } catch (Exception ignored) {}
+ s3Async.close();
+ s3.close();
+ }
+
+ // ── helpers ───────────────────────────────────────────────────────────────
+
+ private void put(String key, String body) {
+ s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(key).build(),
+ RequestBody.fromString(body));
+ }
+
+ private static final InputSerialization CSV_USE = InputSerialization.builder()
+ .csv(CSVInput.builder().fileHeaderInfo(FileHeaderInfo.USE).build()).build();
+ private static final InputSerialization CSV_NONE = InputSerialization.builder()
+ .csv(CSVInput.builder().fileHeaderInfo(FileHeaderInfo.NONE).build()).build();
+ private static final InputSerialization CSV_IGNORE = InputSerialization.builder()
+ .csv(CSVInput.builder().fileHeaderInfo(FileHeaderInfo.IGNORE).build()).build();
+ private static final InputSerialization JSON_IN = InputSerialization.builder()
+ .json(JSONInput.builder().type(JSONType.LINES).build()).build();
+ private static final OutputSerialization CSV_OUT = OutputSerialization.builder()
+ .csv(CSVOutput.builder().build()).build();
+ private static final OutputSerialization JSON_OUT = OutputSerialization.builder()
+ .json(JSONOutput.builder().build()).build();
+
+ private String select(String key, String expression,
+ InputSerialization in, OutputSerialization out) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ CompletableFuture<Void> future = s3Async.selectObjectContent(
+ SelectObjectContentRequest.builder()
+ .bucket(BUCKET).key(key)
+ .expression(expression)
+ .expressionType(ExpressionType.SQL)
+ .inputSerialization(in)
+ .outputSerialization(out)
+ .build(),
+ SelectObjectContentResponseHandler.builder()
+ .subscriber(SelectObjectContentResponseHandler.Visitor.builder()
+ .onRecords(event -> {
+ try {
+ baos.write(event.payload().asByteArray());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .build())
+ .build());
+ future.join();
+ return baos.toString(StandardCharsets.UTF_8);
+ }
+
+ // ── CSV WHERE clause ───────────────────────────────────────────────────────
+
+ @Test
+ @Order(1)
+ @DisplayName("CSV: WHERE with numeric comparison")
+ void csvWhereNumeric() {
+ put("where-num.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ String result = select("where-num.csv", "SELECT * FROM S3Object WHERE age > 30", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Charlie").doesNotContain("Alice").doesNotContain("Bob");
+ }
+
+ @Test
+ @Order(2)
+ @DisplayName("CSV: column projection")
+ void csvProjection() {
+ put("proj.csv", "name,age,city\nAlice,30,New York\nBob,25,London");
+ String result = select("proj.csv", "SELECT name, city FROM S3Object", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Alice,New York").contains("Bob,London")
+ .doesNotContain("30").doesNotContain("25");
+ }
+
+ @Test
+ @Order(3)
+ @DisplayName("CSV: LIMIT")
+ void csvLimit() {
+ put("limit.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ String result = select("limit.csv", "SELECT * FROM S3Object LIMIT 2", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Alice").contains("Bob").doesNotContain("Charlie");
+ }
+
+ // ── String equality / case sensitivity ───────────────────────────────────
+
+ @Test
+ @Order(4)
+ @DisplayName("CSV: string equality matches exact case")
+ void csvStringEquality() {
+ put("case.csv", "name,age\nAlice,30\nBob,25");
+ String result = select("case.csv", "SELECT * FROM S3Object WHERE name = 'Alice'", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Alice").doesNotContain("Bob");
+ }
+
+ @Test
+ @Order(5)
+ @DisplayName("CSV: string equality is case-sensitive")
+ void csvStringEqualityCaseSensitive() {
+ put("case2.csv", "name,age\nAlice,30\nBob,25");
+ String result = select("case2.csv", "SELECT * FROM S3Object WHERE name = 'alice'", CSV_USE, CSV_OUT);
+ assertThat(result).doesNotContain("Alice").doesNotContain("Bob");
+ }
+
+ // ── FileHeaderInfo modes ──────────────────────────────────────────────────
+
+ @Test
+ @Order(6)
+ @DisplayName("CSV: FileHeaderInfo=NONE treats header line as data")
+ void csvFileHeaderInfoNone() {
+ put("none.csv", "name,age\nAlice,30\nBob,25");
+ String result = select("none.csv", "SELECT _1 FROM S3Object", CSV_NONE, CSV_OUT);
+ assertThat(result).contains("name").contains("Alice").contains("Bob");
+ }
+
+ @Test
+ @Order(7)
+ @DisplayName("CSV: FileHeaderInfo=IGNORE skips header row")
+ void csvFileHeaderInfoIgnore() {
+ put("ignore.csv", "name,age\nAlice,30\nBob,25");
+ String result = select("ignore.csv", "SELECT _1 FROM S3Object", CSV_IGNORE, CSV_OUT);
+ assertThat(result).contains("Alice").contains("Bob").doesNotContain("name");
+ }
+
+ // ── Quoted fields ─────────────────────────────────────────────────────────
+
+ @Test
+ @Order(8)
+ @DisplayName("CSV: quoted field with embedded comma is not split")
+ void csvQuotedField() {
+ put("quoted.csv", "name,age\n\"Smith, John\",35\n\"Doe, Jane\",22");
+ String result = select("quoted.csv", "SELECT * FROM S3Object WHERE age > 30", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Smith, John").doesNotContain("Doe, Jane");
+ }
+
+ // ── Enhanced WHERE operators ──────────────────────────────────────────────
+
+ @Test
+ @Order(9)
+ @DisplayName("CSV: WHERE AND")
+ void csvWhereAnd() {
+ put("and.csv", "name,age\nAlice,30\nBob,20\nCharlie,35");
+ String result = select("and.csv", "SELECT * FROM S3Object WHERE age > 20 AND age < 35", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Alice").doesNotContain("Bob").doesNotContain("Charlie");
+ }
+
+ @Test
+ @Order(10)
+ @DisplayName("CSV: WHERE OR")
+ void csvWhereOr() {
+ put("or.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ String result = select("or.csv",
+ "SELECT * FROM S3Object WHERE name = 'Alice' OR name = 'Charlie'", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Alice").contains("Charlie").doesNotContain("Bob");
+ }
+
+ @Test
+ @Order(11)
+ @DisplayName("CSV: WHERE LIKE")
+ void csvWhereLike() {
+ put("like.csv", "name,age\nAlice,30\nBob,25\nAlex,28");
+ String result = select("like.csv", "SELECT * FROM S3Object WHERE name LIKE 'Al%'", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Alice").contains("Alex").doesNotContain("Bob");
+ }
+
+ @Test
+ @Order(12)
+ @DisplayName("CSV: WHERE BETWEEN")
+ void csvWhereBetween() {
+ put("between.csv", "name,age\nAlice,30\nBob,20\nCharlie,25");
+ String result = select("between.csv",
+ "SELECT * FROM S3Object WHERE age BETWEEN 25 AND 30", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Alice").contains("Charlie").doesNotContain("Bob");
+ }
+
+ @Test
+ @Order(13)
+ @DisplayName("CSV: WHERE IN")
+ void csvWhereIn() {
+ put("in.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ String result = select("in.csv",
+ "SELECT * FROM S3Object WHERE name IN ('Alice', 'Charlie')", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Alice").contains("Charlie").doesNotContain("Bob");
+ }
+
+ @Test
+ @Order(14)
+ @DisplayName("CSV: WHERE IS NULL")
+ void csvWhereIsNull() {
+ put("isnull.csv", "name,age,city\nAlice,30,NY\nBob,25");
+ String result = select("isnull.csv",
+ "SELECT name FROM S3Object WHERE city IS NULL", CSV_USE, CSV_OUT);
+ assertThat(result).contains("Bob").doesNotContain("Alice");
+ }
+
+ // ── JSON input ────────────────────────────────────────────────────────────
+
+ @Test
+ @Order(15)
+ @DisplayName("JSON Lines: WHERE clause")
+ void jsonLinesWhere() {
+ put("where.json",
+ "{\"name\":\"Alice\",\"age\":30}\n{\"name\":\"Bob\",\"age\":25}\n{\"name\":\"Charlie\",\"age\":35}");
+ String result = select("where.json", "SELECT * FROM S3Object WHERE age > 30", JSON_IN, JSON_OUT);
+ assertThat(result).contains("Charlie").doesNotContain("Alice").doesNotContain("Bob");
+ }
+
+ @Test
+ @Order(16)
+ @DisplayName("JSON Lines: projection")
+ void jsonLinesProjection() {
+ put("proj.json",
+ "{\"name\":\"Alice\",\"age\":30,\"city\":\"NY\"}\n{\"name\":\"Bob\",\"age\":25,\"city\":\"LA\"}");
+ String result = select("proj.json", "SELECT name FROM S3Object", JSON_IN, JSON_OUT);
+ assertThat(result).contains("Alice").contains("Bob")
+ .doesNotContain("\"age\"").doesNotContain("\"city\"");
+ }
+
+ @Test
+ @Order(17)
+ @DisplayName("JSON Lines: IS NULL")
+ void jsonLinesIsNull() {
+ put("null.json",
+ "{\"name\":\"Alice\",\"age\":30,\"city\":\"NY\"}\n{\"name\":\"Bob\",\"age\":25}");
+ String result = select("null.json",
+ "SELECT name FROM S3Object WHERE city IS NULL", JSON_IN, JSON_OUT);
+ assertThat(result).contains("Bob").doesNotContain("Alice");
+ }
+
+ // ── Cross-format output ───────────────────────────────────────────────────
+
+ @Test
+ @Order(18)
+ @DisplayName("CSV input → JSON output")
+ void csvInputJsonOutput() {
+ put("xfmt.csv", "name,age\nAlice,30\nBob,25");
+ String result = select("xfmt.csv", "SELECT * FROM S3Object", CSV_USE, JSON_OUT);
+ assertThat(result).contains("\"name\"").contains("\"Alice\"").contains("\"Bob\"");
+ }
+
+ @Test
+ @Order(19)
+ @DisplayName("JSON input → CSV output")
+ void jsonInputCsvOutput() {
+ put("xfmt.json", "{\"name\":\"Alice\",\"age\":30}\n{\"name\":\"Bob\",\"age\":25}");
+ String result = select("xfmt.json", "SELECT name, age FROM S3Object", JSON_IN, CSV_OUT);
+ assertThat(result).contains("Alice").contains("Bob");
+ }
+}
diff --git a/docs/services/athena.md b/docs/services/athena.md
index f8f729e7b7..f96827a50b 100644
--- a/docs/services/athena.md
+++ b/docs/services/athena.md
@@ -41,8 +41,8 @@ The DuckDB read function is chosen from the Glue table's `StorageDescriptor`:
| Property | Default | Description |
|---|---|---|
| `FLOCI_SERVICES_ATHENA_MOCK` | `false` | Set to `true` to disable DuckDB execution — queries immediately succeed with empty results |
-| `FLOCI_SERVICES_ATHENA_DEFAULT_IMAGE` | `floci/floci-duck:latest` | DuckDB sidecar image |
-| `FLOCI_SERVICES_ATHENA_DUCK_URL` | *(unset)* | Point to an existing floci-duck instance and skip container management |
+| `FLOCI_SERVICES_DUCK_DEFAULT_IMAGE` | `floci/floci-duck:latest` | DuckDB sidecar image pulled on first use |
+| `FLOCI_SERVICES_DUCK_URL` | *(unset)* | Point to an existing floci-duck instance and skip container management |
## Example — simple query
@@ -116,6 +116,14 @@ done
aws athena get-query-results --query-execution-id $QUERY_ID
```
+## Shared sidecar with S3 Select
+
+The floci-duck sidecar is shared between Athena and S3 Select. Once started by the first Athena query, it is also used by `SelectObjectContent` for CSV (with `FileHeaderInfo=USE`), JSON, and Parquet inputs. If Athena has not yet executed a query, S3 Select falls back to the built-in Java evaluator for CSV and JSON — Parquet always requires the sidecar.
+
+See [S3 Select](s3.md#s3-select) for details on execution modes and supported SQL operators.
+
## Mock mode
-Set `FLOCI_SERVICES_ATHENA_MOCK=true` to skip DuckDB entirely. In this mode queries transition to `SUCCEEDED` immediately with an empty result set — useful for unit tests that only exercise the Athena state machine, not the query results.
+Set `FLOCI_SERVICES_ATHENA_MOCK=true` to skip DuckDB entirely for Athena. In this mode queries transition to `SUCCEEDED` immediately with an empty result set — useful for unit tests that only exercise the Athena state machine, not the query results.
+
+When mock mode is enabled the sidecar does **not** start. S3 Select will use the Java evaluator for CSV and JSON. Parquet queries will fail unless `FLOCI_SERVICES_DUCK_URL` points to an already-running floci-duck instance.
diff --git a/docs/services/s3.md b/docs/services/s3.md
index c7a9f3e8c6..a57423dcdf 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -26,6 +26,103 @@
**RestoreObject** is accepted but stubbed: Floci validates the request and returns `202 Accepted`, but no restore state machine runs.
+## S3 Select
+
+`SelectObjectContent` runs SQL queries directly against S3 objects without downloading the entire file. Floci supports CSV, JSON Lines, JSON arrays, and Parquet inputs. The SQL dialect follows [AWS S3 Select SQL reference](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-select-sql-reference.html).
+
+### Execution modes
+
+Floci chooses the execution engine automatically based on the input format and whether the [floci-duck](https://hub.docker.com/r/floci/floci-duck) sidecar is running:
+
+| Condition | Engine | Notes |
+|---|---|---|
+| Input is Parquet | floci-duck (required) | DuckDB's `read_parquet` — sidecar must be available |
+| Input is CSV with `FileHeaderInfo=USE` and floci-duck is running | floci-duck | Full DuckDB SQL: all operators, LIKE, BETWEEN, IN, IS NULL, AND/OR/NOT |
+| Input is JSON and floci-duck is running | floci-duck | `read_json_auto` — supports JSON Lines and JSON arrays |
+| Input is CSV with `FileHeaderInfo=NONE` or `IGNORE`, or floci-duck is not running | Java evaluator | Supports SELECT *, column projection, simple WHERE with =, !=, <, >, <=, >=, LIKE, BETWEEN, IN, IS NULL, AND/OR/NOT, LIMIT |
+
+The floci-duck sidecar starts lazily on the **first Athena query**. Until then, `isAvailable()` returns false and S3 Select falls back to the Java evaluator for CSV and JSON. Once the sidecar is running, subsequent S3 Select calls route through DuckDB automatically.
+
+If floci-duck is not running and the object is Parquet, S3 Select returns an error — Parquet decoding requires DuckDB.
+
+### FileHeaderInfo modes (CSV)
+
+| Value | Behavior |
+|---|---|
+| `USE` | First row is the header; column names are available in WHERE and SELECT |
+| `IGNORE` | First row is skipped and not included in output; only positional `_N` references work |
+| `NONE` | All rows are data; only positional `_N` references work (e.g. `WHERE _1 = 'Alice'`) |
+
+### Supported SQL operators
+
+When using the Java evaluator (no floci-duck, or CSV with `FileHeaderInfo=NONE/IGNORE`):
+
+- Comparison: `=`, `!=`, `<>`, `<`, `>`, `<=`, `>=`
+- Pattern matching: `LIKE` (supports `%` and `_` wildcards)
+- Range: `BETWEEN ... AND ...`
+- Set membership: `IN (...)`
+- Null checks: `IS NULL`, `IS NOT NULL`
+- Logical: `AND`, `OR`, `NOT`
+- Clauses: `SELECT *`, column projection, `LIMIT`
+
+### Output formats
+
+S3 Select supports cross-format output: a CSV object can produce JSON output and vice versa.
+
+| Input | Output | Format |
+|---|---|---|
+| CSV | CSV | Default — comma-separated values |
+| CSV | JSON | One JSON object per row: `{"col1":"val1","col2":"val2"}` |
+| JSON | JSON | Default — one JSON object per line |
+| JSON | CSV | Comma-separated values, values quoted when they contain commas or newlines |
+
+### Example
+
+```bash
+export AWS_ENDPOINT_URL=http://localhost:4566
+
+# Upload a CSV file
+printf 'name,age,city\nAlice,30,New York\nBob,25,\nCharlie,35,London\n' \
+ | aws s3 cp - s3://my-bucket/people.csv
+
+# Query with WHERE and column projection
+aws s3api select-object-content \
+ --bucket my-bucket \
+ --key people.csv \
+ --expression "SELECT name, city FROM S3Object WHERE age >= 30" \
+ --expression-type SQL \
+ --input-serialization '{"CSV":{"FileHeaderInfo":"USE"}}' \
+ --output-serialization '{"CSV":{}}' \
+ /dev/stdout
+
+# IS NULL check
+aws s3api select-object-content \
+ --bucket my-bucket \
+ --key people.csv \
+ --expression "SELECT name FROM S3Object WHERE city IS NULL" \
+ --expression-type SQL \
+ --input-serialization '{"CSV":{"FileHeaderInfo":"USE"}}' \
+ --output-serialization '{"CSV":{}}' \
+ /dev/stdout
+
+# JSON Lines input
+printf '{"name":"Alice","score":95}\n{"name":"Bob","score":72}\n' \
+ | aws s3 cp - s3://my-bucket/scores.json
+
+aws s3api select-object-content \
+ --bucket my-bucket \
+ --key scores.json \
+ --expression "SELECT * FROM S3Object WHERE score > 80" \
+ --expression-type SQL \
+ --input-serialization '{"JSON":{"Type":"LINES"}}' \
+ --output-serialization '{"JSON":{}}' \
+ /dev/stdout
+```
+
+### Mock mode note
+
+When `FLOCI_SERVICES_ATHENA_MOCK=true` is set, Athena queries are stubbed but floci-duck does **not** start. In that configuration, S3 Select uses the Java evaluator for CSV and JSON. Parquet queries will fail unless `FLOCI_SERVICES_DUCK_URL` points to an already-running floci-duck instance.
+
## Not Implemented
These AWS S3 features have no handler in Floci. Calls will return an error (typically `404` or `NoSuchBucket`-style):
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index da4f07d859..81a2cda437 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -301,6 +301,7 @@ public interface EmulatorConfig {
TransferServiceConfig transfer();
TextractServiceConfig textract();
PricingServiceConfig pricing();
+ DuckConfig duck();
TranscribeServiceConfig transcribe();
CostExplorerServiceConfig ce();
}
@@ -552,9 +553,11 @@ public interface EmulatorConfig {
@WithDefault("false")
boolean mock();
+ }
+ interface DuckConfig {
/** When set, Floci uses this URL and skips floci-duck container management. */
- Optional<String> duckUrl();
+ Optional<String> url();
@WithDefault("floci/floci-duck:latest")
String defaultImage();
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java b/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java
new file mode 100644
index 0000000000..ba993e26eb
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/core/common/CsvParser.java
@@ -0,0 +1,51 @@
+package io.github.hectorvent.floci.core.common;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CsvParser {
+
+ private CsvParser() {}
+
+ public static List<String> parseLine(String line) {
+ List<String> fields = new ArrayList<>();
+ StringBuilder current = new StringBuilder();
+ boolean inQuotes = false;
+ for (int i = 0; i < line.length(); i++) {
+ char c = line.charAt(i);
+ if (inQuotes) {
+ if (c == '"') {
+ if (i + 1 < line.length() && line.charAt(i + 1) == '"') {
+ current.append('"');
+ i++;
+ } else {
+ inQuotes = false;
+ }
+ } else {
+ current.append(c);
+ }
+ } else {
+ if (c == '"') {
+ inQuotes = true;
+ } else if (c == ',') {
+ fields.add(current.toString());
+ current.setLength(0);
+ } else {
+ current.append(c);
+ }
+ }
+ }
+ fields.add(current.toString());
+ return fields;
+ }
+
+ public static List<List<String>> parseAll(String content) {
+ List<List<String>> rows = new ArrayList<>();
+ for (String line : content.split("\\r?\\n")) {
+ if (!line.trim().isEmpty()) {
+ rows.add(parseLine(line));
+ }
+ }
+ return rows;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
index 65c83497a1..eac6828dfd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java
@@ -1,14 +1,13 @@
package io.github.hectorvent.floci.services.athena;
import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
-import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
-import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
+import io.github.hectorvent.floci.core.common.CsvParser;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.athena.model.*;
+import io.github.hectorvent.floci.services.floci.FlociDuckClient;
import io.github.hectorvent.floci.services.glue.GlueService;
import io.github.hectorvent.floci.services.glue.model.Table;
import io.github.hectorvent.floci.services.s3.S3Service;
@@ -21,10 +20,6 @@ import org.jboss.logging.Logger;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.InputStreamReader;
-import java.net.URI;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.*;
@@ -36,37 +31,26 @@ public class AthenaService {
private static final String DEFAULT_OUTPUT_BUCKET = "floci-athena-results";
private final StorageBackend<String, QueryExecution> queryStore;
- private final FlociDuckManager duckManager;
+ private final FlociDuckClient duckClient;
private final GlueService glueService;
private final S3Service s3Service;
private final EmulatorConfig config;
private final Vertx vertx;
- private final ObjectMapper mapper;
- private final HttpClient httpClient;
- private final DockerHostResolver dockerHostResolver;
- private final EmbeddedDnsServer embeddedDnsServer;
@Inject
public AthenaService(StorageFactory storageFactory,
- FlociDuckManager duckManager,
+ FlociDuckClient duckClient,
GlueService glueService,
S3Service s3Service,
EmulatorConfig config,
- Vertx vertx,
- ObjectMapper mapper,
- DockerHostResolver dockerHostResolver,
- EmbeddedDnsServer embeddedDnsServer) {
+ Vertx vertx) {
this.queryStore = storageFactory.create("athena", "queries.json",
- new TypeReference<Map<String, QueryExecution>>() {});
- this.duckManager = duckManager;
+ new TypeReference<>() {});
+ this.duckClient = duckClient;
this.glueService = glueService;
this.s3Service = s3Service;
this.config = config;
this.vertx = vertx;
- this.mapper = mapper;
- this.httpClient = HttpClient.newHttpClient();
- this.dockerHostResolver = dockerHostResolver;
- this.embeddedDnsServer = embeddedDnsServer;
}
public String startQueryExecution(String query,
@@ -95,9 +79,9 @@ public class AthenaService {
// Submit async — caller gets the ID immediately while execution runs in background
String finalDatabase = database;
vertx.executeBlocking(() -> {
- String duckUrl = duckManager.ensureReady();
String setupDdl = buildGlueDdl(finalDatabase);
- callDuck(duckUrl, query, setupDdl, outputLocation, id);
+ ensureOutputBucket(outputLocation);
+ duckClient.execute(query, setupDdl, outputLocation + "results.csv");
return null;
}).onSuccess(v -> {
execution.getStatus().setState(QueryExecutionState.SUCCEEDED);
@@ -197,56 +181,12 @@ public class AthenaService {
return base.endsWith("/") ? base + queryId + "/" : base + "/" + queryId + "/";
}
- private void callDuck(String duckUrl, String sql, String setupDdl, String outputS3Path, String queryId) {
- try {
- // Ensure the output bucket exists
- String bucket = extractBucket(outputS3Path);
- if (bucket != null) {
- try {
- s3Service.createBucket(bucket, config.defaultRegion());
- } catch (Exception ignored) {}
- }
-
- // Floci endpoint reachable from inside the floci-duck container.
- // When the embedded DNS server is active, floci-duck containers already have it wired as their
- // resolver and can reach Floci by the configured hostname (or the default DNS suffix).
- // Fall back to the raw Docker host IP when the embedded DNS is not running (local dev mode).
- int flociPort = URI.create(config.baseUrl()).getPort();
- String flociHostname = embeddedDnsServer.getServerIp().isPresent()
- ? config.hostname().orElse(EmbeddedDnsServer.DEFAULT_SUFFIX)
- : dockerHostResolver.resolve();
- String flociEndpoint = "http://" + flociHostname + ":" + flociPort;
-
- Map<String, Object> body = new LinkedHashMap<>();
- body.put("sql", sql);
- if (setupDdl != null && !setupDdl.isBlank()) {
- body.put("setup_sql", setupDdl);
- }
- body.put("s3_endpoint", flociEndpoint);
- body.put("s3_region", config.defaultRegion());
- body.put("s3_access_key", "test");
- body.put("s3_secret_key", "test");
- body.put("s3_url_style", "path");
- body.put("output_s3_path", outputS3Path + "results.csv");
-
- String json = mapper.writeValueAsString(body);
-
- HttpRequest request = HttpRequest.newBuilder()
- .uri(URI.create(duckUrl + "/execute"))
- .header("Content-Type", "application/json")
- .POST(HttpRequest.BodyPublishers.ofString(json))
- .build();
-
- HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
-
- if (response.statusCode() != 200) {
- throw new RuntimeException("floci-duck returned HTTP " + response.statusCode()
- + ": " + response.body());
- }
- } catch (RuntimeException e) {
- throw e;
- } catch (Exception e) {
- throw new RuntimeException("Failed to call floci-duck for query " + queryId + ": " + e.getMessage(), e);
+ private void ensureOutputBucket(String s3Path) {
+ String bucket = extractBucket(s3Path);
+ if (bucket != null) {
+ try {
+ s3Service.createBucket(bucket, config.defaultRegion());
+ } catch (Exception ignored) {}
}
}
@@ -287,7 +227,7 @@ public class AthenaService {
return emptyResultSet();
}
- String[] headers = splitCsv(headerLine);
+ String[] headers = CsvParser.parseLine(headerLine).toArray(String[]::new);
for (String h : headers) {
columns.add(new ResultSet.ColumnInfo(h, "varchar"));
}
@@ -297,7 +237,7 @@ public class AthenaService {
String line;
while ((line = reader.readLine()) != null) {
- rows.add(toRow(splitCsv(line)));
+ rows.add(toRow(CsvParser.parseLine(line).toArray(String[]::new)));
}
} catch (Exception e) {
LOG.debugv("CSV parse error: {0}", e.getMessage());
@@ -314,31 +254,6 @@ public class AthenaService {
return new ResultSet.Row(data);
}
- /** Minimal CSV split — handles quoted fields. */
- private String[] splitCsv(String line) {
- List<String> fields = new ArrayList<>();
- StringBuilder sb = new StringBuilder();
- boolean inQuotes = false;
- for (int i = 0; i < line.length(); i++) {
- char c = line.charAt(i);
- if (c == '"') {
- if (inQuotes && i + 1 < line.length() && line.charAt(i + 1) == '"') {
- sb.append('"');
- i++;
- } else {
- inQuotes = !inQuotes;
- }
- } else if (c == ',' && !inQuotes) {
- fields.add(sb.toString());
- sb.setLength(0);
- } else {
- sb.append(c);
- }
- }
- fields.add(sb.toString());
- return fields.toArray(new String[0]);
- }
-
private String extractBucket(String s3Path) {
if (s3Path == null || !s3Path.startsWith("s3://")) {
return null;
diff --git a/src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckClient.java b/src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckClient.java
new file mode 100644
index 0000000000..6b7c430de8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckClient.java
@@ -0,0 +1,164 @@
+package io.github.hectorvent.floci.services.floci;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * HTTP client for the floci-duck sidecar.
+ *
+ * Wraps the two duck endpoints used across services:
+ * - /execute — fire-and-forget SQL, writes results to S3 (used by Athena)
+ * - /query — SQL that returns rows as JSON maps (used by S3 Select)
+ */
+@ApplicationScoped
+public class FlociDuckClient {
+
+ private final FlociDuckManager duckManager;
+ private final EmulatorConfig config;
+ private final EmbeddedDnsServer embeddedDnsServer;
+ private final DockerHostResolver dockerHostResolver;
+ private final ObjectMapper mapper;
+ private final HttpClient httpClient;
+
+ @Inject
+ public FlociDuckClient(FlociDuckManager duckManager,
+ EmulatorConfig config,
+ EmbeddedDnsServer embeddedDnsServer,
+ DockerHostResolver dockerHostResolver,
+ ObjectMapper mapper) {
+ this.duckManager = duckManager;
+ this.config = config;
+ this.embeddedDnsServer = embeddedDnsServer;
+ this.dockerHostResolver = dockerHostResolver;
+ this.mapper = mapper;
+ this.httpClient = HttpClient.newHttpClient();
+ }
+
+ /**
+ * Returns true if floci-duck is currently reachable without starting it.
+ */
+ public boolean isAvailable() {
+ return duckManager.isAvailable();
+ }
+
+ /**
+ * Executes SQL via floci-duck and writes the result as a CSV to {@code outputS3Path}.
+ * Used by Athena query execution.
+ *
+ * @param sql the SQL to run (FROM clauses reference s3:// paths)
+ * @param setupDdl optional DDL to run before the query (e.g. CREATE VIEW)
+ * @param outputS3Path the S3 path where the result CSV will be written
+ */
+ public void execute(String sql, String setupDdl, String outputS3Path) {
+ String duckUrl = duckManager.ensureReady();
+
+ Map<String, Object> body = buildBaseBody(sql, setupDdl);
+ body.put("output_s3_path", outputS3Path);
+
+ post(duckUrl + "/execute", body, "execute");
+ }
+
+ /**
+ * Executes SQL via floci-duck and returns the result rows as a list of column→value maps.
+ * Used by S3 Select for formats that require DuckDB evaluation (e.g. Parquet).
+ *
+ * @param sql the SQL to run (FROM clauses reference s3:// paths)
+ * @param setupDdl optional DDL to run before the query
+ * @return result rows; empty list if the query matched no rows
+ */
+ public List<Map<String, Object>> query(String sql, String setupDdl) {
+ String duckUrl = duckManager.ensureReady();
+
+ Map<String, Object> body = buildBaseBody(sql, setupDdl);
+
+ String responseBody = post(duckUrl + "/query", body, "query");
+ return parseQueryRows(responseBody);
+ }
+
+ // ── internals ─────────────────────────────────────────────────────────────
+
+ private Map<String, Object> buildBaseBody(String sql, String setupDdl) {
+ Map<String, Object> body = new LinkedHashMap<>();
+ body.put("sql", sql);
+ if (setupDdl != null && !setupDdl.isBlank()) {
+ body.put("setup_sql", setupDdl);
+ }
+ body.put("s3_endpoint", resolveFlociEndpoint());
+ body.put("s3_region", config.defaultRegion());
+ body.put("s3_access_key", "test");
+ body.put("s3_secret_key", "test");
+ body.put("s3_url_style", "path");
+ return body;
+ }
+
+ /**
+ * Returns the Floci S3 endpoint URL as reachable from inside a floci-duck container.
+ * Uses the embedded DNS hostname when the DNS server is active, falls back to the
+ * raw Docker host IP for local dev mode.
+ */
+ private String resolveFlociEndpoint() {
+ int port = URI.create(config.baseUrl()).getPort();
+ String hostname = embeddedDnsServer.getServerIp().isPresent()
+ ? config.hostname().orElse(EmbeddedDnsServer.DEFAULT_SUFFIX)
+ : dockerHostResolver.resolve();
+ return "http://" + hostname + ":" + port;
+ }
+
+ private String post(String url, Map<String, Object> body, String operation) {
+ try {
+ String json = mapper.writeValueAsString(body);
+ HttpRequest request = HttpRequest.newBuilder()
+ .uri(URI.create(url))
+ .header("Content-Type", "application/json")
+ .POST(HttpRequest.BodyPublishers.ofString(json))
+ .build();
+ HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+ if (response.statusCode() != 200) {
+ throw new RuntimeException("floci-duck " + operation + " returned HTTP "
+ + response.statusCode() + ": " + response.body());
+ }
+ return response.body();
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to call floci-duck " + operation + ": " + e.getMessage(), e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<Map<String, Object>> parseQueryRows(String responseBody) {
+ try {
+ JsonNode root = mapper.readTree(responseBody);
+ if (!"success".equals(root.path("status").asText())) {
+ throw new RuntimeException("floci-duck query error: " + root.path("message").asText());
+ }
+ JsonNode rowsNode = root.path("rows");
+ if (rowsNode.isMissingNode() || rowsNode.isNull()) {
+ return List.of();
+ }
+ List<Map<String, Object>> rows = new ArrayList<>();
+ for (JsonNode row : rowsNode) {
+ rows.add((Map<String, Object>) mapper.treeToValue(row, Map.class));
+ }
+ return rows;
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to parse floci-duck query response: " + e.getMessage(), e);
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/FlociDuckManager.java b/src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckManager.java
similarity index 79%
rename from src/main/java/io/github/hectorvent/floci/services/athena/FlociDuckManager.java
rename to src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckManager.java
index a2d3834f17..8c93aa2bc6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/athena/FlociDuckManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/floci/FlociDuckManager.java
@@ -1,4 +1,4 @@
-package io.github.hectorvent.floci.services.athena;
+package io.github.hectorvent.floci.services.floci;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
@@ -23,7 +23,7 @@ import java.util.Optional;
* On the first call to {@link #ensureReady()}, Floci pulls the image and starts
* a named container "floci-duck". Subsequent calls return the cached URL immediately.
*
- * If {@code floci.services.athena.duck-url} is configured, container management is
+ * If {@code floci.services.duck.url} is configured, container management is
* skipped entirely and that URL is used as-is — useful in Docker Compose setups where
* the user runs floci-duck as a separate service.
*/
@@ -55,6 +55,37 @@ public class FlociDuckManager {
this.config = config;
}
+ /**
+ * Returns true if floci-duck is reachable without starting anything.
+ * If a URL is pre-configured and healthy, that URL is cached for future ensureReady() calls.
+ */
+ public synchronized boolean isAvailable() {
+ if (resolvedUrl != null) {
+ return true;
+ }
+ Optional<String> configured = config.services().duck().url();
+ if (configured.isPresent() && !configured.get().isBlank()) {
+ String url = configured.get();
+ if (probeHealth(url)) {
+ resolvedUrl = url;
+ LOG.infov("floci-duck is available at pre-configured URL: {0}", url);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean probeHealth(String baseUrl) {
+ try {
+ HttpURLConnection conn = (HttpURLConnection) URI.create(baseUrl + "/health").toURL().openConnection();
+ conn.setConnectTimeout(500);
+ conn.setReadTimeout(500);
+ return conn.getResponseCode() == 200;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
/**
* Returns the floci-duck base URL, starting the container on first call if needed.
* Thread-safe — concurrent callers wait while the first thread does the pull+start.
@@ -64,7 +95,7 @@ public class FlociDuckManager {
return resolvedUrl;
}
- Optional<String> configured = config.services().athena().duckUrl();
+ Optional<String> configured = config.services().duck().url();
if (configured.isPresent() && !configured.get().isBlank()) {
resolvedUrl = configured.get();
LOG.infov("Using pre-configured floci-duck URL: {0}", resolvedUrl);
@@ -76,7 +107,7 @@ public class FlociDuckManager {
}
private void startContainer() {
- String image = config.services().athena().defaultImage();
+ String image = config.services().duck().defaultImage();
LOG.infov("Starting floci-duck container using image {0}", image);
lifecycleManager.removeIfExists(CONTAINER_NAME);
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.java
index 9bfa4bd671..4512eb0ee7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.java
@@ -1,14 +1,18 @@
package io.github.hectorvent.floci.services.s3;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.CsvParser;
+import org.jboss.logging.Logger;
+
+import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import org.jboss.logging.Logger;
+import java.util.stream.StreamSupport;
-public class S3SelectEvaluator {
+class S3SelectEvaluator {
private static final Logger LOG = Logger.getLogger(S3SelectEvaluator.class);
@@ -16,13 +20,195 @@ public class S3SelectEvaluator {
"SELECT\\s+(.+?)\\s+FROM\\s+S3OBJECT(?:\\s+(\\w+))?\\s*(?:WHERE\\s+(.+?))?\\s*(?:LIMIT\\s+(\\d+))?\\s*$",
Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
- public static String evaluateCsv(String content, String expression, boolean useHeaders) {
+ // ── AST node types ─────────────────────────────────────────────────────
+
+ private sealed interface WhereNode
+ permits AndNode, OrNode, NotNode, CompNode, IsNullNode, BetweenNode, InNode, LikeNode {}
+
+ private record AndNode(WhereNode left, WhereNode right) implements WhereNode {}
+ private record OrNode(WhereNode left, WhereNode right) implements WhereNode {}
+ private record NotNode(WhereNode child) implements WhereNode {}
+ private record CompNode(String col, String op, String val) implements WhereNode {}
+ private record IsNullNode(String col, boolean negate) implements WhereNode {}
+ private record BetweenNode(String col, String lo, String hi) implements WhereNode {}
+ private record InNode(String col, List<String> vals) implements WhereNode {}
+ private record LikeNode(String col, String pattern) implements WhereNode {}
+
+ // ── Lexer ──────────────────────────────────────────────────────────────
+
+ private enum TokenType { IDENT, STRING, NUMBER, OP, KEYWORD, LPAREN, RPAREN, COMMA }
+
+ private record Token(TokenType type, String value) {}
+
+ private static final Set<String> KEYWORDS = Set.of(
+ "AND", "OR", "NOT", "LIKE", "BETWEEN", "IN", "IS", "NULL", "TRUE", "FALSE"
+ );
+
+ private static List<Token> tokenize(String input) {
+ List<Token> tokens = new ArrayList<>();
+ int i = 0;
+ while (i < input.length()) {
+ char c = input.charAt(i);
+ if (Character.isWhitespace(c)) { i++; continue; }
+ if (c == '(') { tokens.add(new Token(TokenType.LPAREN, "(")); i++; continue; }
+ if (c == ')') { tokens.add(new Token(TokenType.RPAREN, ")")); i++; continue; }
+ if (c == ',') { tokens.add(new Token(TokenType.COMMA, ",")); i++; continue; }
+
+ if (c == '\'') {
+ StringBuilder sb = new StringBuilder();
+ i++;
+ while (i < input.length()) {
+ char ch = input.charAt(i);
+ if (ch == '\'' && i + 1 < input.length() && input.charAt(i + 1) == '\'') {
+ sb.append('\''); i += 2;
+ } else if (ch == '\'') {
+ i++; break;
+ } else {
+ sb.append(ch); i++;
+ }
+ }
+ tokens.add(new Token(TokenType.STRING, sb.toString()));
+ continue;
+ }
+
+ if (Character.isDigit(c) || (c == '-' && i + 1 < input.length() && Character.isDigit(input.charAt(i + 1)))) {
+ int start = i++;
+ while (i < input.length() && (Character.isDigit(input.charAt(i)) || input.charAt(i) == '.')) i++;
+ tokens.add(new Token(TokenType.NUMBER, input.substring(start, i)));
+ continue;
+ }
+
+ if (i + 1 < input.length()) {
+ String two = input.substring(i, i + 2);
+ if (">=".equals(two) || "<=".equals(two) || "<>".equals(two) || "!=".equals(two)) {
+ tokens.add(new Token(TokenType.OP, two)); i += 2; continue;
+ }
+ }
+ if (c == '>' || c == '<' || c == '=') {
+ tokens.add(new Token(TokenType.OP, String.valueOf(c))); i++; continue;
+ }
+
+ if (Character.isLetter(c) || c == '_') {
+ int start = i++;
+ while (i < input.length() && (Character.isLetterOrDigit(input.charAt(i))
+ || input.charAt(i) == '_' || input.charAt(i) == '.')) i++;
+ String word = input.substring(start, i);
+ String upper = word.toUpperCase();
+ if (KEYWORDS.contains(upper)) {
+ tokens.add(new Token(TokenType.KEYWORD, upper));
+ } else {
+ tokens.add(new Token(TokenType.IDENT, word));
+ }
+ continue;
+ }
+ i++;
+ }
+ return tokens;
+ }
+
+ // ── Recursive descent parser ───────────────────────────────────────────
+
+ private static final class Parser {
+ private final List<Token> tokens;
+ private int pos;
+
+ Parser(List<Token> tokens) { this.tokens = tokens; }
+
+ WhereNode parse() { return orExpr(); }
+
+ private WhereNode orExpr() {
+ WhereNode left = andExpr();
+ while (peekKeyword("OR")) { advance(); left = new OrNode(left, andExpr()); }
+ return left;
+ }
+
+ private WhereNode andExpr() {
+ WhereNode left = notExpr();
+ while (peekKeyword("AND")) { advance(); left = new AndNode(left, notExpr()); }
+ return left;
+ }
+
+ private WhereNode notExpr() {
+ if (peekKeyword("NOT")) { advance(); return new NotNode(notExpr()); }
+ return atom();
+ }
+
+ private WhereNode atom() {
+ if (pos < tokens.size() && tokens.get(pos).type() == TokenType.LPAREN) {
+ advance();
+ WhereNode inner = orExpr();
+ if (pos < tokens.size() && tokens.get(pos).type() == TokenType.RPAREN) advance();
+ return inner;
+ }
+ String col = consumeOperand();
+ if (pos >= tokens.size()) return new CompNode(col, "=", "");
+ Token next = tokens.get(pos);
+ if (next.type() == TokenType.KEYWORD) {
+ return switch (next.value()) {
+ case "IS" -> {
+ advance();
+ boolean negate = peekKeyword("NOT");
+ if (negate) advance();
+ if (peekKeyword("NULL")) advance();
+ yield new IsNullNode(col, negate);
+ }
+ case "BETWEEN" -> {
+ advance();
+ String lo = consumeOperand();
+ if (peekKeyword("AND")) advance();
+ yield new BetweenNode(col, lo, consumeOperand());
+ }
+ case "IN" -> {
+ advance();
+ if (pos < tokens.size() && tokens.get(pos).type() == TokenType.LPAREN) advance();
+ List<String> vals = new ArrayList<>();
+ while (pos < tokens.size() && tokens.get(pos).type() != TokenType.RPAREN) {
+ vals.add(consumeOperand());
+ if (pos < tokens.size() && tokens.get(pos).type() == TokenType.COMMA) advance();
+ }
+ if (pos < tokens.size() && tokens.get(pos).type() == TokenType.RPAREN) advance();
+ yield new InNode(col, vals);
+ }
+ case "LIKE" -> {
+ advance();
+ yield new LikeNode(col, consumeOperand());
+ }
+ default -> new CompNode(col, "=", "");
+ };
+ }
+ if (next.type() == TokenType.OP) {
+ String op = next.value(); advance();
+ return new CompNode(col, op, consumeOperand());
+ }
+ return new CompNode(col, "=", "");
+ }
+
+ private String consumeOperand() {
+ if (pos >= tokens.size()) return "";
+ return tokens.get(pos++).value();
+ }
+
+ private boolean peekKeyword(String kw) {
+ return pos < tokens.size()
+ && tokens.get(pos).type() == TokenType.KEYWORD
+ && kw.equals(tokens.get(pos).value());
+ }
+
+ private void advance() { if (pos < tokens.size()) pos++; }
+ }
+
+ private static WhereNode parseWhere(String where) {
+ return new Parser(tokenize(where)).parse();
+ }
+
+ // ── CSV evaluation ─────────────────────────────────────────────────────
+
+ static String evaluateCsv(String content, String expression, String fileHeaderInfo, String outputFormat) {
Matcher matcher = SELECT_PATTERN.matcher(expression.trim());
if (!matcher.find()) {
- LOG.debugv("SQL Pattern did not match: {0}", expression);
+ LOG.debugv("SQL pattern did not match: {0}", expression);
return content;
}
-
String projection = matcher.group(1).trim();
String alias = matcher.group(2);
String whereClause = matcher.group(3);
@@ -33,104 +219,312 @@ public class S3SelectEvaluator {
List<String> headerList = new ArrayList<>();
int dataStart = 0;
- if (useHeaders) {
- headerList = Arrays.asList(lines[0].split(","));
+ if ("USE".equalsIgnoreCase(fileHeaderInfo)) {
+ headerList = CsvParser.parseLine(lines[0]);
+ dataStart = 1;
+ } else if ("IGNORE".equalsIgnoreCase(fileHeaderInfo)) {
dataStart = 1;
}
List<String[]> rows = new ArrayList<>();
for (int i = dataStart; i < lines.length; i++) {
if (lines[i].trim().isEmpty()) continue;
- rows.add(lines[i].split(","));
+ rows.add(CsvParser.parseLine(lines[i]).toArray(String[]::new));
}
- // Filter
if (whereClause != null) {
- rows = filterRows(rows, headerList, alias, whereClause);
+ rows = filterCsvRows(rows, headerList, alias, whereClause);
}
- // Limit
if (limitStr != null) {
int limit = Integer.parseInt(limitStr);
- if (rows.size() > limit) {
- rows = rows.subList(0, limit);
- }
+ if (rows.size() > limit) rows = rows.subList(0, limit);
}
- // Project
- return projectRows(rows, headerList, projection);
+ return projectCsvRows(rows, headerList, projection, outputFormat);
}
- private static List<String[]> filterRows(List<String[]> rows, List<String> headers, String alias, String where) {
- String processedWhere = where;
+ private static List<String[]> filterCsvRows(List<String[]> rows, List<String> headers,
+ String alias, String where) {
+ String processed = where;
if (alias != null) {
- processedWhere = processedWhere.replaceAll("(?i)" + Pattern.quote(alias) + "\\.", "");
- }
- final String finalWhere = processedWhere;
-
- return rows.stream().filter(row -> {
- // Simple evaluation: col > val or col = val
- // Try to find a comparison
- Pattern compPattern = Pattern.compile("(\\w+)\\s*(>|=|<)\\s*(\\d+|'[^']*')", Pattern.CASE_INSENSITIVE);
- Matcher m = compPattern.matcher(finalWhere);
- if (m.find()) {
- String colName = m.group(1);
- String op = m.group(2);
- String valStr = m.group(3);
-
- String cellValue = getCellValue(row, headers, colName);
- if (cellValue == null) return false;
-
- if (valStr.startsWith("'")) {
- String val = valStr.substring(1, valStr.length() - 1);
- return op.equals("=") && cellValue.equalsIgnoreCase(val);
- } else {
- try {
- double cellNum = Double.parseDouble(cellValue);
- double valNum = Double.parseDouble(valStr);
- return switch (op) {
- case ">" -> cellNum > valNum;
- case "<" -> cellNum < valNum;
- case "=" -> cellNum == valNum;
- default -> false;
- };
- } catch (NumberFormatException e) {
- return false;
- }
- }
+ processed = processed.replaceAll("(?i)" + Pattern.quote(alias) + "\\.", "");
+ }
+ try {
+ WhereNode tree = parseWhere(processed);
+ return rows.stream().filter(row -> evalCsv(tree, row, headers)).toList();
+ } catch (Exception e) {
+ LOG.warnv("WHERE parse error, returning all rows: {0}", e.getMessage());
+ return rows;
+ }
+ }
+
+ private static boolean evalCsv(WhereNode node, String[] row, List<String> headers) {
+ return switch (node) {
+ case AndNode(var l, var r) -> evalCsv(l, row, headers) && evalCsv(r, row, headers);
+ case OrNode(var l, var r) -> evalCsv(l, row, headers) || evalCsv(r, row, headers);
+ case NotNode(var child) -> !evalCsv(child, row, headers);
+ case IsNullNode(var col, var negate) -> {
+ String v = getCellValue(row, headers, col);
+ yield negate ? v != null : v == null;
+ }
+ case BetweenNode(var col, var lo, var hi) -> {
+ String v = getCellValue(row, headers, col);
+ yield v != null && cmp(v, lo) >= 0 && cmp(v, hi) <= 0;
+ }
+ case InNode(var col, var vals) -> {
+ String v = getCellValue(row, headers, col);
+ yield v != null && vals.stream().anyMatch(v::equals);
}
- return true;
- }).collect(Collectors.toList());
+ case LikeNode(var col, var pattern) -> {
+ String v = getCellValue(row, headers, col);
+ yield v != null && matchLike(v, pattern);
+ }
+ case CompNode(var col, var op, var val) -> {
+ String v = getCellValue(row, headers, col);
+ yield v != null && compareOp(v, op, val);
+ }
+ };
}
- private static String projectRows(List<String[]> rows, List<String> headers, String projection) {
- if (projection.equals("*")) {
+ private static String projectCsvRows(List<String[]> rows, List<String> headers,
+ String projection, String outputFormat) {
+ boolean toJson = "JSON".equalsIgnoreCase(outputFormat);
+ if ("*".equals(projection)) {
return rows.stream()
- .map(row -> String.join(",", row))
+ .map(row -> toJson ? rowToJson(row, headers) : String.join(",", row))
.collect(Collectors.joining("\n")) + (rows.isEmpty() ? "" : "\n");
}
-
- String[] cols = projection.split(",");
+ String[] cols = Arrays.stream(projection.split(",")).map(String::trim).toArray(String[]::new);
return rows.stream().map(row -> {
- List<String> projected = new ArrayList<>();
- for (String col : cols) {
- projected.add(getCellValue(row, headers, col.trim()));
+ List<String> vals = Arrays.stream(cols)
+ .map(col -> { String v = getCellValue(row, headers, col); return v != null ? v : ""; })
+ .toList();
+ if (toJson) {
+ StringBuilder json = new StringBuilder("{");
+ for (int i = 0; i < cols.length; i++) {
+ if (i > 0) json.append(",");
+ json.append('"').append(jsonEscape(cols[i])).append("\":\"")
+ .append(jsonEscape(vals.get(i))).append('"');
+ }
+ return json.append("}").toString();
}
- return String.join(",", projected);
+ return String.join(",", vals);
}).collect(Collectors.joining("\n")) + (rows.isEmpty() ? "" : "\n");
}
+ // ── Duck rows formatting ───────────────────────────────────────────────
+
+ /**
+ * Serializes rows returned by floci-duck (/query) into the requested output format.
+ * DuckDB already applied filtering and projection, so this is pure serialization.
+ */
+ static String formatDuckRows(List<Map<String, Object>> rows, String outputFormat, ObjectMapper mapper) {
+ boolean toJson = "JSON".equalsIgnoreCase(outputFormat);
+ StringBuilder sb = new StringBuilder();
+ for (Map<String, Object> row : rows) {
+ if (toJson) {
+ try {
+ sb.append(mapper.writeValueAsString(row)).append("\n");
+ } catch (Exception ignored) {}
+ } else {
+ sb.append(row.values().stream()
+ .map(v -> v == null ? "" : csvEscape(v.toString()))
+ .collect(Collectors.joining(","))).append("\n");
+ }
+ }
+ return sb.toString();
+ }
+
+ // ── JSON evaluation ────────────────────────────────────────────────────
+
+ static String evaluateJson(String content, String expression, ObjectMapper mapper, String outputFormat) {
+ Matcher matcher = SELECT_PATTERN.matcher(expression.trim());
+ if (!matcher.find()) {
+ LOG.debugv("SQL pattern did not match for JSON: {0}", expression);
+ return content;
+ }
+ String projection = matcher.group(1).trim();
+ String alias = matcher.group(2);
+ String whereClause = matcher.group(3);
+ String limitStr = matcher.group(4);
+
+ WhereNode tree = null;
+ if (whereClause != null) {
+ String processed = whereClause;
+ if (alias != null) {
+ processed = processed.replaceAll("(?i)" + Pattern.quote(alias) + "\\.", "");
+ }
+ try {
+ tree = parseWhere(processed);
+ } catch (Exception e) {
+ LOG.warnv("JSON WHERE parse error: {0}", e.getMessage());
+ }
+ }
+
+ List<JsonNode> rows = parseJsonInput(content, mapper);
+
+ if (tree != null) {
+ final WhereNode finalTree = tree;
+ rows = rows.stream().filter(row -> evalJson(finalTree, row)).toList();
+ }
+
+ if (limitStr != null) {
+ int limit = Integer.parseInt(limitStr);
+ if (rows.size() > limit) rows = rows.subList(0, limit);
+ }
+
+ return projectJsonRows(rows, projection, outputFormat, mapper);
+ }
+
+ private static List<JsonNode> parseJsonInput(String content, ObjectMapper mapper) {
+ // Try JSON array first (starts with '['), then JSON Lines, then single object.
+ if (content.trim().startsWith("[")) {
+ try {
+ JsonNode root = mapper.readTree(content);
+ if (root.isArray()) {
+ return StreamSupport.stream(root.spliterator(), false).toList();
+ }
+ } catch (Exception ignored) {}
+ }
+ // JSON Lines: parse each non-empty line independently.
+ List<JsonNode> rows = new ArrayList<>();
+ for (String line : content.split("\\r?\\n")) {
+ if (line.trim().isEmpty()) continue;
+ try { rows.add(mapper.readTree(line)); } catch (Exception ignored) {}
+ }
+ return rows;
+ }
+
+ private static boolean evalJson(WhereNode node, JsonNode row) {
+ return switch (node) {
+ case AndNode(var l, var r) -> evalJson(l, row) && evalJson(r, row);
+ case OrNode(var l, var r) -> evalJson(l, row) || evalJson(r, row);
+ case NotNode(var child) -> !evalJson(child, row);
+ case IsNullNode(var col, var negate) -> {
+ JsonNode v = row.path(col);
+ boolean isNull = v.isMissingNode() || v.isNull();
+ yield negate ? !isNull : isNull;
+ }
+ case BetweenNode(var col, var lo, var hi) -> {
+ String v = row.path(col).asText(null);
+ yield v != null && cmp(v, lo) >= 0 && cmp(v, hi) <= 0;
+ }
+ case InNode(var col, var vals) -> {
+ String v = row.path(col).asText(null);
+ yield v != null && vals.stream().anyMatch(v::equals);
+ }
+ case LikeNode(var col, var pattern) -> {
+ String v = row.path(col).asText(null);
+ yield v != null && matchLike(v, pattern);
+ }
+ case CompNode(var col, var op, var val) -> {
+ String v = row.path(col).asText(null);
+ yield v != null && compareOp(v, op, val);
+ }
+ };
+ }
+
+ private static String projectJsonRows(List<JsonNode> rows, String projection,
+ String outputFormat, ObjectMapper mapper) {
+ boolean toCsv = "CSV".equalsIgnoreCase(outputFormat);
+ String[] cols = "*".equals(projection) ? null
+ : Arrays.stream(projection.split(",")).map(String::trim).toArray(String[]::new);
+ StringBuilder sb = new StringBuilder();
+ for (JsonNode row : rows) {
+ if (cols == null) {
+ if (toCsv) {
+ List<String> vals = new ArrayList<>();
+ row.fields().forEachRemaining(e -> vals.add(csvEscape(e.getValue().asText(""))));
+ sb.append(String.join(",", vals)).append("\n");
+ } else {
+ try { sb.append(mapper.writeValueAsString(row)).append("\n"); } catch (Exception ignored) {}
+ }
+ } else {
+ if (toCsv) {
+ sb.append(Arrays.stream(cols)
+ .map(col -> csvEscape(row.path(col).asText("")))
+ .collect(Collectors.joining(","))).append("\n");
+ } else {
+ ObjectNode projected = mapper.createObjectNode();
+ for (String col : cols) {
+ JsonNode v = row.path(col);
+ if (!v.isMissingNode()) projected.set(col, v);
+ }
+ try { sb.append(mapper.writeValueAsString(projected)).append("\n"); } catch (Exception ignored) {}
+ }
+ }
+ }
+ return sb.toString();
+ }
+
+ // ── Shared helpers ─────────────────────────────────────────────────────
+
+ private static boolean compareOp(String cell, String op, String val) {
+ int c = cmp(cell, val);
+ return switch (op) {
+ case "=" -> c == 0;
+ case "<>", "!=" -> c != 0;
+ case ">" -> c > 0;
+ case "<" -> c < 0;
+ case ">=" -> c >= 0;
+ case "<=" -> c <= 0;
+ default -> false;
+ };
+ }
+
+ private static int cmp(String a, String b) {
+ try {
+ return Double.compare(Double.parseDouble(a), Double.parseDouble(b));
+ } catch (NumberFormatException e) {
+ return a.compareTo(b);
+ }
+ }
+
+ private static boolean matchLike(String value, String pattern) {
+ StringBuilder regex = new StringBuilder("^");
+ for (int i = 0; i < pattern.length(); i++) {
+ char c = pattern.charAt(i);
+ if (c == '%') regex.append(".*");
+ else if (c == '_') regex.append(".");
+ else regex.append(Pattern.quote(String.valueOf(c)));
+ }
+ return Pattern.compile(regex.append("$").toString(), Pattern.DOTALL).matcher(value).matches();
+ }
+
+ private static String rowToJson(String[] row, List<String> headers) {
+ StringBuilder json = new StringBuilder("{");
+ for (int i = 0; i < headers.size(); i++) {
+ if (i > 0) json.append(",");
+ String val = i < row.length ? row[i] : "";
+ json.append('"').append(jsonEscape(headers.get(i).trim())).append("\":\"")
+ .append(jsonEscape(val)).append('"');
+ }
+ return json.append("}").toString();
+ }
+
+ private static String jsonEscape(String s) {
+ return s.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n").replace("\r", "\\r");
+ }
+
+ private static String csvEscape(String s) {
+ if (s.contains(",") || s.contains("\"") || s.contains("\n")) {
+ return "\"" + s.replace("\"", "\"\"") + "\"";
+ }
+ return s;
+ }
+
private static String getCellValue(String[] row, List<String> headers, String colName) {
if (colName.startsWith("_")) {
try {
int idx = Integer.parseInt(colName.substring(1)) - 1;
- return idx < row.length ? row[idx] : null;
+ return idx >= 0 && idx < row.length ? row[idx] : null;
} catch (NumberFormatException e) {
return null;
}
}
for (int i = 0; i < headers.size(); i++) {
- if (headers.get(i).equalsIgnoreCase(colName)) {
+ if (headers.get(i).trim().equalsIgnoreCase(colName)) {
return i < row.length ? row[i] : null;
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.java
index 6f9f7ad541..50144b011b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.java
@@ -2,62 +2,123 @@ package io.github.hectorvent.floci.services.s3;
import io.github.hectorvent.floci.core.common.AwsEventStreamEncoder;
import io.github.hectorvent.floci.core.common.XmlParser;
+import io.github.hectorvent.floci.services.floci.FlociDuckClient;
import io.github.hectorvent.floci.services.s3.model.S3Object;
-import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
@ApplicationScoped
public class S3SelectService {
+ private static final Pattern S3OBJECT_PATTERN =
+ Pattern.compile("\\bS3Object\\b", Pattern.CASE_INSENSITIVE);
+
private final ObjectMapper objectMapper;
+ private final FlociDuckClient duckClient;
@Inject
- public S3SelectService(ObjectMapper objectMapper) {
+ public S3SelectService(ObjectMapper objectMapper, FlociDuckClient duckClient) {
this.objectMapper = objectMapper;
+ this.duckClient = duckClient;
}
public byte[] select(S3Object object, String requestXml) {
- String expression = XmlParser.extractFirst(requestXml, "Expression", "").toUpperCase();
- String inputType = requestXml.contains("<CSV>") ? "CSV" : (requestXml.contains("<JSON>") ? "JSON" : null);
-
+ String expression = XmlParser.extractFirst(requestXml, "Expression", "");
+ String fileHeaderInfo = XmlParser.extractFirst(requestXml, "FileHeaderInfo", "NONE");
+ String inputType = detectType(requestXml, "InputSerialization");
+ String outputFormat = detectType(requestXml, "OutputSerialization");
+ if (outputFormat == null) outputFormat = "CSV";
+
byte[] rawData = object.getData();
if (rawData == null) return new byte[0];
- String content = new String(rawData, StandardCharsets.UTF_8);
-
- StringBuilder result = new StringBuilder();
-
- if ("CSV".equals(inputType)) {
- boolean useHeaders = requestXml.contains("<FileHeaderInfo>USE</FileHeaderInfo>");
- String filtered = S3SelectEvaluator.evaluateCsv(content, expression, useHeaders);
- result.append(filtered);
+ long bytesScanned = rawData.length;
+
+ String result;
+ if (isParquet(object)) {
+ result = selectParquet(object, expression, outputFormat);
+ } else if (duckClient.isAvailable() && canUseDuck(inputType, fileHeaderInfo)) {
+ result = selectViaDuck(object, expression, inputType, outputFormat);
+ } else if ("CSV".equals(inputType)) {
+ String content = new String(rawData, StandardCharsets.UTF_8);
+ result = S3SelectEvaluator.evaluateCsv(content, expression, fileHeaderInfo, outputFormat);
} else if ("JSON".equals(inputType)) {
- // Assume one JSON object per line (JSON Lines) or a single array
- try {
- JsonNode node = objectMapper.readTree(content);
- if (node.isArray()) {
- for (JsonNode item : node) {
- result.append(objectMapper.writeValueAsString(item)).append("\n");
- }
- } else {
- result.append(objectMapper.writeValueAsString(node)).append("\n");
- }
- } catch (Exception e) {
- // If it's not valid JSON, just return raw or fail
- result.append(content);
- }
+ String content = new String(rawData, StandardCharsets.UTF_8);
+ result = S3SelectEvaluator.evaluateJson(content, expression, objectMapper, outputFormat);
} else {
- result.append(content);
+ result = new String(rawData, StandardCharsets.UTF_8);
}
- return encodeEventStream(result.toString());
+ long bytesReturned = result.getBytes(StandardCharsets.UTF_8).length;
+ return encodeEventStream(result, bytesScanned, bytesReturned);
+ }
+
+ // ── Duck delegation ────────────────────────────────────────────────────
+
+ /**
+ * Duck handles CSV with USE headers (named columns) and all JSON.
+ * CSV NONE/IGNORE use _N positional refs that DuckDB doesn't natively support,
+ * so those fall back to the Java evaluator.
+ */
+ private static boolean canUseDuck(String inputType, String fileHeaderInfo) {
+ if ("JSON".equals(inputType)) {
+ return true;
+ }
+ return "CSV".equals(inputType) && "USE".equals(fileHeaderInfo);
+ }
+
+ private String selectViaDuck(S3Object object, String expression, String inputType,
+ String outputFormat) {
+ String s3Uri = "s3://" + object.getBucketName() + "/" + object.getKey();
+ String readFn = "JSON".equals(inputType)
+ ? "read_json_auto('" + s3Uri + "')"
+ : "read_csv('" + s3Uri + "', header=true, null_padding=true)";
+ String duckSql = S3OBJECT_PATTERN.matcher(expression)
+ .replaceAll(Matcher.quoteReplacement(readFn));
+ List<Map<String, Object>> rows = duckClient.query(duckSql, null);
+ return S3SelectEvaluator.formatDuckRows(rows, outputFormat, objectMapper);
+ }
+
+ // ── Parquet delegation ─────────────────────────────────────────────────
+
+ private static boolean isParquet(S3Object object) {
+ String ct = object.getContentType();
+ if (ct != null && ct.toLowerCase().contains("parquet")) return true;
+ String key = object.getKey();
+ return key != null && key.toLowerCase().endsWith(".parquet");
+ }
+
+ private String selectParquet(S3Object object, String expression, String outputFormat) {
+ String s3Uri = "s3://" + object.getBucketName() + "/" + object.getKey();
+ String duckSql = S3OBJECT_PATTERN.matcher(expression)
+ .replaceAll(Matcher.quoteReplacement("read_parquet('" + s3Uri + "')"));
+
+ List<Map<String, Object>> rows = duckClient.query(duckSql, null);
+ return S3SelectEvaluator.formatDuckRows(rows, outputFormat, objectMapper);
+ }
+
+ // ── Helpers ────────────────────────────────────────────────────────────
+
+ private static String detectType(String requestXml, String sectionTag) {
+ String openTag = "<" + sectionTag + ">";
+ int start = requestXml.indexOf(openTag);
+ if (start < 0) return null;
+ int end = requestXml.indexOf("</" + sectionTag + ">", start);
+ String section = end > start ? requestXml.substring(start, end) : requestXml.substring(start);
+ if (section.contains("<CSV")) return "CSV";
+ if (section.contains("<JSON")) return "JSON";
+ if (section.contains("<Parquet")) return "PARQUET";
+ return null;
}
- private byte[] encodeEventStream(String payload) {
+ private byte[] encodeEventStream(String payload, long bytesScanned, long bytesReturned) {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -71,8 +132,10 @@ public class S3SelectService {
statsHeaders.put(":message-type", "event");
statsHeaders.put(":event-type", "Stats");
statsHeaders.put(":content-type", "text/xml");
- byte[] statsPayload = "<Stats><BytesScanned>100</BytesScanned><BytesProcessed>100</BytesProcessed><BytesReturned>100</BytesReturned></Stats>".getBytes(StandardCharsets.UTF_8);
- baos.write(AwsEventStreamEncoder.encodeMessage(statsHeaders, statsPayload));
+ String statsXml = "<Stats><BytesScanned>" + bytesScanned + "</BytesScanned>"
+ + "<BytesProcessed>" + bytesScanned + "</BytesProcessed>"
+ + "<BytesReturned>" + bytesReturned + "</BytesReturned></Stats>";
+ baos.write(AwsEventStreamEncoder.encodeMessage(statsHeaders, statsXml.getBytes(StandardCharsets.UTF_8)));
LinkedHashMap<String, String> endHeaders = new LinkedHashMap<>();
endHeaders.put(":message-type", "event");
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 432c385fd7..b621b92bc1 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -187,7 +187,8 @@ floci:
athena:
enabled: true
mock: false
- # duck-url: http://floci-duck:3000 # set to skip container management
+ duck:
+ # url: http://floci-duck:3000 # set to skip container management
default-image: "floci/floci-duck:latest"
glue:
enabled: true
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.java
index 6b40a1d70d..6a346a220a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.java
@@ -3,8 +3,6 @@ package io.github.hectorvent.floci.services.s3;
import io.quarkus.test.junit.QuarkusTest;
import org.junit.jupiter.api.Test;
-import java.util.Base64;
-
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.not;
@@ -12,124 +10,311 @@ import static org.hamcrest.Matchers.not;
@QuarkusTest
class S3SelectIntegrationTest {
+ // ── helpers ───────────────────────────────────────────────────────────
+
+ private static void createBucketAndPut(String bucket, String key, String body) {
+ given().header("Host", bucket + ".localhost").when().put("/").then().statusCode(200);
+ given().header("Host", bucket + ".localhost").body(body).when().put("/" + key).then().statusCode(200);
+ }
+
+ private static io.restassured.response.ValidatableResponse select(
+ String bucket, String key, String expression,
+ String inputSerialization, String outputSerialization) {
+ // XML-escape the expression so operators like < and & are valid inside <Expression>
+ String safeExpr = expression.replace("&", "&amp;").replace("<", "&lt;");
+ String requestXml = """
+ <?xml version="1.0" encoding="UTF-8"?>
+ <SelectObjectContentRequest>
+ <Expression>%s</Expression>
+ <ExpressionType>SQL</ExpressionType>
+ <InputSerialization>%s</InputSerialization>
+ <OutputSerialization>%s</OutputSerialization>
+ </SelectObjectContentRequest>
+ """.formatted(safeExpr, inputSerialization, outputSerialization);
+ return given()
+ .header("Host", bucket + ".localhost")
+ .queryParam("select", "")
+ .queryParam("select-type", "2")
+ .body(requestXml)
+ .when().post("/" + key)
+ .then().statusCode(200);
+ }
+
+ private static final String CSV_OUT = "<CSV/>";
+ private static final String JSON_OUT = "<JSON/>";
+ private static final String CSV_USE = "<CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV>";
+ private static final String CSV_NONE = "<CSV><FileHeaderInfo>NONE</FileHeaderInfo></CSV>";
+ private static final String CSV_IGNORE = "<CSV><FileHeaderInfo>IGNORE</FileHeaderInfo></CSV>";
+ private static final String JSON_IN = "<JSON/>";
+
+ // ── Existing tests (preserved) ────────────────────────────────────────
+
@Test
void select_withWhereClause() {
String bucket = "select-bucket";
- String key = "data.csv";
- String csvData = "name,age\nAlice,30\nBob,25\nCharlie,35";
-
- // 1. Create bucket
- given()
- .header("Host", bucket + ".localhost")
- .when()
- .put("/")
- .then()
- .statusCode(200);
-
- // 2. Put object
- given()
- .header("Host", bucket + ".localhost")
- .body(csvData)
- .when()
- .put("/" + key)
- .then()
- .statusCode(200);
-
- // 3. Select with WHERE clause
- String requestXml = """
- <?xml version="1.0" encoding="UTF-8"?>
- <SelectObjectContentRequest>
- <Expression>SELECT * FROM S3Object s WHERE s.age > 30</Expression>
- <ExpressionType>SQL</ExpressionType>
- <InputSerialization>
- <CSV>
- <FileHeaderInfo>USE</FileHeaderInfo>
- </CSV>
- </InputSerialization>
- <OutputSerialization>
- <CSV/>
- </OutputSerialization>
- </SelectObjectContentRequest>
- """;
-
- given()
- .header("Host", bucket + ".localhost")
- .queryParam("select", "")
- .queryParam("select-type", "2")
- .body(requestXml)
- .when()
- .post("/" + key)
- .then()
- .statusCode(200)
- .body(containsString("Charlie,35"))
- .body(not(containsString("Alice,30")))
- .body(not(containsString("Bob,25")));
+ createBucketAndPut(bucket, "data.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ select(bucket, "data.csv", "SELECT * FROM S3Object s WHERE s.age > 30", CSV_USE, CSV_OUT)
+ .body(containsString("Charlie,35"))
+ .body(not(containsString("Alice,30")))
+ .body(not(containsString("Bob,25")));
}
@Test
void select_withProjection() {
String bucket = "select-bucket-proj";
- String key = "data.csv";
- String csvData = "name,age,city\nAlice,30,New York\nBob,25,London";
-
- given().header("Host", bucket + ".localhost").when().put("/").then().statusCode(200);
- given().header("Host", bucket + ".localhost").body(csvData).when().put("/" + key).then().statusCode(200);
-
- String requestXml = """
- <?xml version="1.0" encoding="UTF-8"?>
- <SelectObjectContentRequest>
- <Expression>SELECT name, city FROM S3Object</Expression>
- <ExpressionType>SQL</ExpressionType>
- <InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>
- <OutputSerialization><CSV/></OutputSerialization>
- </SelectObjectContentRequest>
- """;
-
- given()
- .header("Host", bucket + ".localhost")
- .queryParam("select", "")
- .queryParam("select-type", "2")
- .body(requestXml)
- .when()
- .post("/" + key)
- .then()
- .statusCode(200)
- .body(containsString("Alice,New York"))
- .body(containsString("Bob,London"))
- .body(not(containsString("30")))
- .body(not(containsString("25")));
+ createBucketAndPut(bucket, "data.csv", "name,age,city\nAlice,30,New York\nBob,25,London");
+ select(bucket, "data.csv", "SELECT name, city FROM S3Object", CSV_USE, CSV_OUT)
+ .body(containsString("Alice,New York"))
+ .body(containsString("Bob,London"))
+ .body(not(containsString("30")))
+ .body(not(containsString("25")));
}
@Test
void select_withLimit() {
String bucket = "select-bucket-limit";
- String key = "data.csv";
- String csvData = "name,age\nAlice,30\nBob,25\nCharlie,35";
+ createBucketAndPut(bucket, "data.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ select(bucket, "data.csv", "SELECT * FROM S3Object LIMIT 2", CSV_USE, CSV_OUT)
+ .body(containsString("Alice,30"))
+ .body(containsString("Bob,25"))
+ .body(not(containsString("Charlie,35")));
+ }
- given().header("Host", bucket + ".localhost").when().put("/").then().statusCode(200);
- given().header("Host", bucket + ".localhost").body(csvData).when().put("/" + key).then().statusCode(200);
+ // ── A: String literal / toUpperCase fix ───────────────────────────────
- String requestXml = """
- <?xml version="1.0" encoding="UTF-8"?>
- <SelectObjectContentRequest>
- <Expression>SELECT * FROM S3Object LIMIT 2</Expression>
- <ExpressionType>SQL</ExpressionType>
- <InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>
- <OutputSerialization><CSV/></OutputSerialization>
- </SelectObjectContentRequest>
- """;
-
- given()
- .header("Host", bucket + ".localhost")
- .queryParam("select", "")
- .queryParam("select-type", "2")
- .body(requestXml)
- .when()
- .post("/" + key)
- .then()
- .statusCode(200)
- .body(containsString("Alice,30"))
- .body(containsString("Bob,25"))
- .body(not(containsString("Charlie,35")));
+ @Test
+ void select_whereStringEquality() {
+ String bucket = "sel-str-eq";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE name = 'Alice'", CSV_USE, CSV_OUT)
+ .body(containsString("Alice,30"))
+ .body(not(containsString("Bob")))
+ .body(not(containsString("Charlie")));
+ }
+
+ @Test
+ void select_whereStringEquality_caseSensitive() {
+ String bucket = "sel-str-case";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25");
+ // 'alice' (lowercase) must NOT match 'Alice' — S3 Select = is case-sensitive
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE name = 'alice'", CSV_USE, CSV_OUT)
+ .body(not(containsString("Alice")))
+ .body(not(containsString("Bob")));
+ }
+
+ // ── B: FileHeaderInfo ─────────────────────────────────────────────────
+
+ @Test
+ void select_fileHeaderInfoNone() {
+ String bucket = "sel-hdr-none";
+ // With NONE, the first line is treated as data, positional refs must be used
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25");
+ select(bucket, "d.csv", "SELECT _1 FROM S3Object", CSV_NONE, CSV_OUT)
+ // header line itself appears as a data row
+ .body(containsString("name"))
+ .body(containsString("Alice"))
+ .body(containsString("Bob"));
+ }
+
+ @Test
+ void select_fileHeaderInfoIgnore() {
+ String bucket = "sel-hdr-ignore";
+ // With IGNORE, header row is skipped from output; positional refs work for data rows
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25");
+ select(bucket, "d.csv", "SELECT _1 FROM S3Object", CSV_IGNORE, CSV_OUT)
+ .body(containsString("Alice"))
+ .body(containsString("Bob"))
+ .body(not(containsString("name"))); // header row excluded
+ }
+
+ // ── C: Quoted CSV fields ──────────────────────────────────────────────
+
+ @Test
+ void select_quotedCsvField() {
+ String bucket = "sel-quoted";
+ // "Smith, John" contains a comma inside quotes — must not be split
+ createBucketAndPut(bucket, "d.csv", "name,age\n\"Smith, John\",35\n\"Doe, Jane\",22");
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age > 30", CSV_USE, CSV_OUT)
+ .body(containsString("Smith, John"))
+ .body(not(containsString("Doe, Jane")));
+ }
+
+ // ── D: Enhanced WHERE operators ───────────────────────────────────────
+
+ @Test
+ void select_whereAnd() {
+ String bucket = "sel-and";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,20\nCharlie,35");
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age > 20 AND age < 35", CSV_USE, CSV_OUT)
+ .body(containsString("Alice,30"))
+ .body(not(containsString("Bob")))
+ .body(not(containsString("Charlie")));
+ }
+
+ @Test
+ void select_whereOr() {
+ String bucket = "sel-or";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE name = 'Alice' OR name = 'Charlie'", CSV_USE, CSV_OUT)
+ .body(containsString("Alice"))
+ .body(containsString("Charlie"))
+ .body(not(containsString("Bob")));
+ }
+
+ @Test
+ void select_whereNot() {
+ String bucket = "sel-not";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE NOT age = 30", CSV_USE, CSV_OUT)
+ .body(not(containsString("Alice")))
+ .body(containsString("Bob,25"))
+ .body(containsString("Charlie,35"));
+ }
+
+ @Test
+ void select_whereLike() {
+ String bucket = "sel-like";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nAlex,28");
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE name LIKE 'Al%'", CSV_USE, CSV_OUT)
+ .body(containsString("Alice"))
+ .body(containsString("Alex"))
+ .body(not(containsString("Bob")));
+ }
+
+ @Test
+ void select_whereIsNull() {
+ String bucket = "sel-isnull";
+ // Bob's row has no city field → city IS NULL
+ createBucketAndPut(bucket, "d.csv", "name,age,city\nAlice,30,NY\nBob,25");
+ select(bucket, "d.csv", "SELECT name FROM S3Object WHERE city IS NULL", CSV_USE, CSV_OUT)
+ .body(containsString("Bob"))
+ .body(not(containsString("Alice")));
+ }
+
+ @Test
+ void select_whereBetween() {
+ String bucket = "sel-between";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,20\nCharlie,25");
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age BETWEEN 25 AND 30", CSV_USE, CSV_OUT)
+ .body(containsString("Alice,30"))
+ .body(containsString("Charlie,25"))
+ .body(not(containsString("Bob,20")));
+ }
+
+ @Test
+ void select_whereIn() {
+ String bucket = "sel-in";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE name IN ('Alice', 'Charlie')", CSV_USE, CSV_OUT)
+ .body(containsString("Alice"))
+ .body(containsString("Charlie"))
+ .body(not(containsString("Bob")));
+ }
+
+ @Test
+ void select_whereGreaterOrEqual() {
+ String bucket = "sel-gte";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age >= 30", CSV_USE, CSV_OUT)
+ .body(containsString("Alice,30")) // exactly 30 must be included
+ .body(containsString("Charlie,35"))
+ .body(not(containsString("Bob,25")));
+ }
+
+ @Test
+ void select_whereLessOrEqual() {
+ String bucket = "sel-lte";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25\nCharlie,35");
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age <= 25", CSV_USE, CSV_OUT)
+ .body(containsString("Bob,25")) // exactly 25 must be included
+ .body(not(containsString("Alice,30")))
+ .body(not(containsString("Charlie,35")));
+ }
+
+ // ── E: JSON input evaluation ──────────────────────────────────────────
+
+ @Test
+ void select_jsonLinesWithWhere() {
+ String bucket = "sel-json-where";
+ String jsonLines = "{\"name\":\"Alice\",\"age\":30}\n{\"name\":\"Bob\",\"age\":25}\n{\"name\":\"Charlie\",\"age\":35}";
+ createBucketAndPut(bucket, "d.json", jsonLines);
+ select(bucket, "d.json", "SELECT * FROM S3Object WHERE age > 30", JSON_IN, JSON_OUT)
+ .body(containsString("Charlie"))
+ .body(not(containsString("Alice")))
+ .body(not(containsString("Bob")));
}
+
+ @Test
+ void select_jsonLinesWithProjection() {
+ String bucket = "sel-json-proj";
+ String jsonLines = "{\"name\":\"Alice\",\"age\":30,\"city\":\"NY\"}\n{\"name\":\"Bob\",\"age\":25,\"city\":\"LA\"}";
+ createBucketAndPut(bucket, "d.json", jsonLines);
+ select(bucket, "d.json", "SELECT name FROM S3Object", JSON_IN, JSON_OUT)
+ .body(containsString("Alice"))
+ .body(containsString("Bob"))
+ // check JSON key absence rather than raw substring to avoid binary CRC false-positives
+ .body(not(containsString("\"age\":")))
+ .body(not(containsString("\"city\":")));
+ }
+
+ @Test
+ void select_jsonArrayWithWhere() {
+ String bucket = "sel-json-arr";
+ String jsonArray = "[{\"name\":\"Alice\",\"age\":30},{\"name\":\"Bob\",\"age\":25}]";
+ createBucketAndPut(bucket, "d.json", jsonArray);
+ select(bucket, "d.json", "SELECT * FROM S3Object WHERE age < 30", JSON_IN, JSON_OUT)
+ .body(containsString("Bob"))
+ .body(not(containsString("Alice")));
+ }
+
+ @Test
+ void select_jsonLinesIsNull() {
+ String bucket = "sel-json-null";
+ String jsonLines = "{\"name\":\"Alice\",\"age\":30,\"city\":\"NY\"}\n{\"name\":\"Bob\",\"age\":25}";
+ createBucketAndPut(bucket, "d.json", jsonLines);
+ select(bucket, "d.json", "SELECT name FROM S3Object WHERE city IS NULL", JSON_IN, JSON_OUT)
+ .body(containsString("Bob"))
+ .body(not(containsString("Alice")));
+ }
+
+ // ── F: Output format ──────────────────────────────────────────────────
+
+ @Test
+ void select_csvInputJsonOutput() {
+ String bucket = "sel-csv-json-out";
+ createBucketAndPut(bucket, "d.csv", "name,age\nAlice,30\nBob,25");
+ select(bucket, "d.csv", "SELECT * FROM S3Object", CSV_USE, JSON_OUT)
+ .body(containsString("\"name\""))
+ .body(containsString("\"Alice\""))
+ .body(containsString("\"Bob\""));
+ }
+
+ @Test
+ void select_jsonInputCsvOutput() {
+ String bucket = "sel-json-csv-out";
+ String jsonLines = "{\"name\":\"Alice\",\"age\":30}\n{\"name\":\"Bob\",\"age\":25}";
+ createBucketAndPut(bucket, "d.json", jsonLines);
+ select(bucket, "d.json", "SELECT name, age FROM S3Object", JSON_IN, CSV_OUT)
+ .body(containsString("Alice,30"))
+ .body(containsString("Bob,25"));
+ }
+
+ // ── G: Real stats bytes ───────────────────────────────────────────────
+
+ @Test
+ void select_statsReflectsActualBytes() {
+ String bucket = "sel-stats";
+ String csvData = "name,age\nAlice,30\nBob,25\nCharlie,35";
+ createBucketAndPut(bucket, "d.csv", csvData);
+ // WHERE filters out 2 of 3 rows, so BytesReturned < BytesScanned
+ select(bucket, "d.csv", "SELECT * FROM S3Object WHERE age > 30", CSV_USE, CSV_OUT)
+ // Stats XML is embedded as UTF-8 in the binary event stream
+ .body(containsString("<BytesScanned>"))
+ .body(containsString("<BytesReturned>"))
+ // Must not be the old hardcoded value
+ .body(not(containsString("<BytesScanned>100</BytesScanned>")));
+ }
+
+ // Note: Parquet S3 Select tests (group H) require Docker + floci-duck and
+ // are covered in compatibility-tests/ where the full stack is available.
}