Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): run S3 Select over CSV and JSON a record at a time (#5178)
Priamo Germosena0a2a5f0d6fd
diff --git a/docs/services/s3.md b/docs/services/s3.mdindex 0838ad882e..d107c7c8a9 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -125,6 +125,15 @@ The floci-duck sidecar starts lazily on the **first Athena query**. Until then,If floci-duck is not running and the object is Parquet, S3 Select returns an error — Parquet decoding requires DuckDB.+The Java evaluator reads the object a record at a time and writes the result in `Records` events as+it goes, stopping once a `LIMIT` is reached, so neither the object nor the result is held in memory.+As on S3, a record can be split across `Records` events, and a record in the input or the result+can be at most 1 MB (1,048,576 characters): a longer one fails the query with `OverMaxRecordSize`,+which arrives as an error message at the end of the event stream once the response has begun.+`BytesScanned` and `BytesProcessed` in the `Stats` event count the bytes the query read, so one a+`LIMIT` stops early reports less than the object's size. A query floci-duck runs is read by DuckDB,+and its result is held in memory before it is returned.+### FileHeaderInfo modes (CSV)| Value | Behavior |@@ -409,8 +418,8 @@ is rejected with `EntityTooLarge`. In the disk-backed modes`CopyObject` stores the source's file for the copy without reading it into memory, and reads itonce only when the copy needs an ETag or checksum the source's cannot supply. As on S3, a`CopyObject` source larger than 5 GiB is rejected with `InvalidRequest`; copy it in parts with-`UploadPartCopy` instead. S3 Select still reads the whole source object into memory in every-mode, so it needs a source under 2 GiB.+`UploadPartCopy` instead. S3 Select's Java evaluator reads a CSV or JSON object a record at a time,+so it takes a source of any size; see [S3 Select](#s3-select).## Examplesdiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 933e532800..2d8a9f9595 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -1530,9 +1530,12 @@ public class S3Controller {if (hasQueryParam(uriInfo, "select")) {s3Service.authorizeGetObject(bucket, key, versionId, authorization);- S3Object obj = s3Service.getObject(bucket, key, versionId);- byte[] result = s3SelectService.select(obj, new String(body, StandardCharsets.UTF_8));- return Response.ok(result)+ // The object and its metadata are opened as one snapshot, as for GetObject, and the+ // query reads it as the response is written, so the object is never held in memory.+ S3SelectService.EventStream events = s3SelectService.select(+ s3Service.openObject(bucket, key, versionId), new String(body, StandardCharsets.UTF_8));+ StreamingOutput entity = events::writeTo;+ return Response.ok(entity).type("application/octet-stream").build();}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.javaindex bf763da3cc..a49f746bde 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectEvaluator.java@@ -1,5 +1,8 @@package io.github.hectorvent.floci.services.s3;+import com.fasterxml.jackson.core.JsonParser;+import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.core.JsonToken;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;@@ -7,16 +10,25 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.CsvParser;import org.jboss.logging.Logger;+import java.io.BufferedReader;+import java.io.FilterReader;+import java.io.IOException;+import java.io.Reader;import java.util.*;import java.util.regex.Matcher;import java.util.regex.Pattern;import java.util.stream.Collectors;-import java.util.stream.StreamSupport;class S3SelectEvaluator {private static final Logger LOG = Logger.getLogger(S3SelectEvaluator.class);+ // S3 Select's limit on the length of a record in the input or the result, its maxCharsPerRecord of 1 MB.+ static final int MAX_RECORD_CHARS = 1024 * 1024;+ // A JSON parser reads ahead of the element it is on; past the record limit by this much, the+ // element is too long whatever the read-ahead holds.+ private static final int PARSER_READ_AHEAD = 64 * 1024;+private static final Pattern SELECT_PATTERN = Pattern.compile("SELECT\\s+(.+?)\\s+FROM\\s+S3OBJECT(?:\\s+(\\w+))?\\s*(?:WHERE\\s+(.+?))?\\s*(?:LIMIT\\s+(\\d+))?\\s*$",Pattern.CASE_INSENSITIVE | Pattern.DOTALL);@@ -206,6 +218,11 @@ class S3SelectEvaluator {private void advance() { if (pos < tokens.size()) pos++; }}+ static AwsException overMaxRecordSize() {+ return new AwsException("OverMaxRecordSize",+ "The length of a record in the input or result is greater than the maxCharsPerRecord limit of 1 MB.", 400);+ }+private static AwsException unsupportedWhereExpression(String expression) {return new AwsException("ExternalEvalException",@@ -218,63 +235,98 @@ class S3SelectEvaluator {return new Parser(tokenize(where)).parse();}- // ── CSV evaluation ─────────────────────────────────────────────────────+ // ── Query ──────────────────────────────────────────────────────────────++ /** A parsed {@code SELECT ... FROM S3Object} statement, checked before any of the object is read. */+ static final class Query {+ private final String[] columns;+ private final WhereNode where;+ private final Integer limit;++ private Query(String[] columns, WhereNode where, Integer limit) {+ this.columns = columns;+ this.where = where;+ this.limit = limit;+ }++ private boolean limitReached(int selected) {+ return limit != null && selected >= limit;+ }+ }++ /** Where an evaluation hands each selected record, without its line break. */+ @FunctionalInterface+ interface RecordSink {+ void accept(String record) throws IOException;+ }- static String evaluateCsv(String content, String expression, String fileHeaderInfo, String outputFormat) {+ /**+ * Parses {@code expression}, or returns {@code null} for one this evaluator does not read, whose+ * object is then returned as it is. A WHERE clause it cannot evaluate fails here.+ */+ static Query parse(String expression) {Matcher matcher = SELECT_PATTERN.matcher(expression.trim());if (!matcher.find()) {LOG.debugv("SQL pattern did not match: {0}", expression);- return content;+ return null;}String projection = matcher.group(1).trim();String alias = matcher.group(2);String whereClause = matcher.group(3);String limitStr = matcher.group(4);- String[] lines = content.split("\\r?\\n");- if (lines.length == 0) return "";-- List<String> headerList = new ArrayList<>();- int dataStart = 0;- if ("USE".equalsIgnoreCase(fileHeaderInfo)) {- headerList = CsvParser.parseLine(lines[0]);- dataStart = 1;- } else if ("IGNORE".equalsIgnoreCase(fileHeaderInfo)) {- dataStart = 1;- }-- List<String[]> rows = new ArrayList<>();- for (int i = dataStart; i < lines.length; i++) {- if (lines[i].trim().isEmpty()) continue;- rows.add(CsvParser.parseLine(lines[i]).toArray(String[]::new));- }-+ WhereNode where = null;if (whereClause != null) {- rows = filterCsvRows(rows, headerList, alias, whereClause);- }-- if (limitStr != null) {- int limit = Integer.parseInt(limitStr);- if (rows.size() > limit) rows = rows.subList(0, limit);+ String processed = whereClause;+ if (alias != null) {+ processed = processed.replaceAll("(?i)" + Pattern.quote(alias) + "\\.", "");+ }+ try {+ where = parseWhere(processed);+ } catch (AwsException e) {+ throw e;+ } catch (Exception e) {+ LOG.warnv("WHERE parse error: {0}", e.getMessage());+ throw unsupportedWhereExpression(processed);+ }}-- return projectCsvRows(rows, headerList, projection, outputFormat);+ String[] columns = "*".equals(projection) ? null+ : Arrays.stream(projection.split(",")).map(String::trim).toArray(String[]::new);+ return new Query(columns, where, limitStr != null ? Integer.parseInt(limitStr) : null);}- private static List<String[]> filterCsvRows(List<String[]> rows, List<String> headers,- String alias, String where) {- String processed = where;- if (alias != null) {- processed = processed.replaceAll("(?i)" + Pattern.quote(alias) + "\\.", "");+ // ── CSV evaluation ─────────────────────────────────────────────────────++ /**+ * Evaluates {@code query} over CSV read from {@code in} one line at a time, handing each selected+ * record to {@code out}, and stops reading once the LIMIT is reached.+ */+ static void evaluateCsv(BufferedReader reader, Query query, String fileHeaderInfo, String outputFormat,+ RecordSink out) throws IOException {+ LineReader in = new LineReader(reader);+ List<String> headers = new ArrayList<>();+ if ("USE".equalsIgnoreCase(fileHeaderInfo) || "IGNORE".equalsIgnoreCase(fileHeaderInfo)) {+ String header = in.readLine();+ if (header == null) {+ return;+ }+ if ("USE".equalsIgnoreCase(fileHeaderInfo)) {+ headers = CsvParser.parseLine(header);+ }}- try {- WhereNode tree = parseWhere(processed);- return rows.stream().filter(row -> evalCsv(tree, row, headers)).toList();- } catch (AwsException e) {- throw e;- } catch (Exception e) {- LOG.warnv("WHERE parse error: {0}", e.getMessage());- throw unsupportedWhereExpression(processed);+ boolean toJson = "JSON".equalsIgnoreCase(outputFormat);+ int selected = 0;+ String line;+ while (!query.limitReached(selected) && (line = in.readLine()) != null) {+ if (line.trim().isEmpty()) {+ continue;+ }+ String[] row = CsvParser.parseLine(line).toArray(String[]::new);+ if (query.where != null && !evalCsv(query.where, row, headers)) {+ continue;+ }+ out.accept(projectCsvRow(row, headers, query.columns, toJson));+ selected++;}}@@ -306,30 +358,28 @@ class S3SelectEvaluator {};}- private static String projectCsvRows(List<String[]> rows, List<String> headers,- String projection, String outputFormat) {- boolean toJson = "JSON".equalsIgnoreCase(outputFormat);- if ("*".equals(projection)) {- return rows.stream()- .map(row -> toJson ? rowToJson(row, headers) : String.join(",", row))- .collect(Collectors.joining("\n")) + (rows.isEmpty() ? "" : "\n");- }- String[] cols = Arrays.stream(projection.split(",")).map(String::trim).toArray(String[]::new);- return rows.stream().map(row -> {- List<String> vals = Arrays.stream(cols)- .map(col -> { String v = getCellValue(row, headers, col); return v != null ? v : ""; })- .toList();- if (toJson) {- StringBuilder json = new StringBuilder("{");- for (int i = 0; i < cols.length; i++) {- if (i > 0) json.append(",");- json.append('"').append(jsonEscape(cols[i])).append("\":\"")- .append(jsonEscape(vals.get(i))).append('"');+ private static String projectCsvRow(String[] row, List<String> headers, String[] columns, boolean toJson) {+ if (columns == null) {+ return toJson ? rowToJson(row, headers) : String.join(",", row);+ }+ List<String> vals = Arrays.stream(columns)+ .map(col -> {+ String v = getCellValue(row, headers, col);+ return v != null ? v : "";+ })+ .toList();+ if (toJson) {+ StringBuilder json = new StringBuilder("{");+ for (int i = 0; i < columns.length; i++) {+ if (i > 0) {+ json.append(",");}- return json.append("}").toString();+ json.append('"').append(jsonEscape(columns[i])).append("\":\"")+ .append(jsonEscape(vals.get(i))).append('"');}- return String.join(",", vals);- }).collect(Collectors.joining("\n")) + (rows.isEmpty() ? "" : "\n");+ return json.append("}").toString();+ }+ return String.join(",", vals);}// ── Duck rows formatting ───────────────────────────────────────────────@@ -357,65 +407,81 @@ class S3SelectEvaluator {// ── JSON evaluation ────────────────────────────────────────────────────- static String evaluateJson(String content, String expression, ObjectMapper mapper, String outputFormat) {- Matcher matcher = SELECT_PATTERN.matcher(expression.trim());- if (!matcher.find()) {- LOG.debugv("SQL pattern did not match for JSON: {0}", expression);- return content;+ /**+ * Evaluates {@code query} over JSON read from {@code in}, a record at a time: the elements of a+ * top-level array, or else one JSON value per line. Each selected record goes to {@code out}, and+ * reading stops once the LIMIT is reached.+ */+ static void evaluateJson(BufferedReader in, Query query, ObjectMapper mapper, String outputFormat,+ RecordSink out) throws IOException {+ boolean toCsv = "CSV".equalsIgnoreCase(outputFormat);+ int selected = 0;+ if (startsWithArray(in)) {+ ElementLimitedReader limited = new ElementLimitedReader(in);+ JsonParser parser = mapper.getFactory().createParser(limited);+ try {+ parser.nextToken();+ while (!query.limitReached(selected)) {+ JsonToken token = parser.nextToken();+ if (token == null || token == JsonToken.END_ARRAY) {+ break;+ }+ limited.startElement();+ long start = parser.currentTokenLocation().getCharOffset();+ JsonNode row = mapper.readTree(parser);+ limited.endElement();+ if (parser.currentLocation().getCharOffset() - start > MAX_RECORD_CHARS) {+ throw overMaxRecordSize();+ }+ if (query.where == null || evalJson(query.where, row)) {+ out.accept(projectJsonRow(row, query.columns, toCsv, mapper));+ selected++;+ }+ }+ } catch (JsonProcessingException e) {+ // An element the parser gave up on past the record limit is over it, whatever stopped+ // the parser: a number, for one, it reads whole before checking its own length limit.+ if (limited.overRecordLimit()) {+ throw overMaxRecordSize();+ }+ // The records before the malformed part have been returned; the rest cannot be read.+ LOG.debugv("Stopped reading a malformed JSON array for S3 Select: {0}", e.getOriginalMessage());+ }+ return;}- String projection = matcher.group(1).trim();- String alias = matcher.group(2);- String whereClause = matcher.group(3);- String limitStr = matcher.group(4);-- WhereNode tree = null;- if (whereClause != null) {- String processed = whereClause;- if (alias != null) {- processed = processed.replaceAll("(?i)" + Pattern.quote(alias) + "\\.", "");+ LineReader lines = new LineReader(in);+ String line;+ while (!query.limitReached(selected) && (line = lines.readLine()) != null) {+ if (line.trim().isEmpty()) {+ continue;}+ JsonNode row;try {- tree = parseWhere(processed);- } catch (AwsException e) {- throw e;- } catch (Exception e) {- LOG.warnv("JSON WHERE parse error: {0}", e.getMessage());- throw unsupportedWhereExpression(processed);+ row = mapper.readTree(line);+ } catch (JsonProcessingException ignored) {+ // A line that is not JSON is not a record; it is skipped, as it always has been.+ continue;+ }+ if (query.where == null || evalJson(query.where, row)) {+ out.accept(projectJsonRow(row, query.columns, toCsv, mapper));+ selected++;}}-- List<JsonNode> rows = parseJsonInput(content, mapper);-- if (tree != null) {- final WhereNode finalTree = tree;- rows = rows.stream().filter(row -> evalJson(finalTree, row)).toList();- }-- if (limitStr != null) {- int limit = Integer.parseInt(limitStr);- if (rows.size() > limit) rows = rows.subList(0, limit);- }-- return projectJsonRows(rows, projection, outputFormat, mapper);}- private static List<JsonNode> parseJsonInput(String content, ObjectMapper mapper) {- // Try JSON array first (starts with '['), then JSON Lines, then single object.- if (content.trim().startsWith("[")) {- try {- JsonNode root = mapper.readTree(content);- if (root.isArray()) {- return StreamSupport.stream(root.spliterator(), false).toList();- }- } catch (Exception ignored) {}- }- // JSON Lines: parse each non-empty line independently.- List<JsonNode> rows = new ArrayList<>();- for (String line : content.split("\\r?\\n")) {- if (line.trim().isEmpty()) continue;- try { rows.add(mapper.readTree(line)); } catch (Exception ignored) {}+ /** Whether the first character after any leading whitespace opens an array; nothing is consumed. */+ private static boolean startsWithArray(BufferedReader in) throws IOException {+ while (true) {+ in.mark(1);+ int c = in.read();+ if (c < 0) {+ return false;+ }+ if (!Character.isWhitespace(c)) {+ in.reset();+ return c == '[';+ }}- return rows;}private static boolean evalJson(WhereNode node, JsonNode row) {@@ -447,37 +513,146 @@ class S3SelectEvaluator {};}- private static String projectJsonRows(List<JsonNode> rows, String projection,- String outputFormat, ObjectMapper mapper) {- boolean toCsv = "CSV".equalsIgnoreCase(outputFormat);- String[] cols = "*".equals(projection) ? null- : Arrays.stream(projection.split(",")).map(String::trim).toArray(String[]::new);- StringBuilder sb = new StringBuilder();- for (JsonNode row : rows) {- if (cols == null) {- if (toCsv) {- List<String> vals = new ArrayList<>();- row.fields().forEachRemaining(e -> vals.add(csvEscape(e.getValue().asText(""))));- sb.append(String.join(",", vals)).append("\n");- } else {- try { sb.append(mapper.writeValueAsString(row)).append("\n"); } catch (Exception ignored) {}+ private static String projectJsonRow(JsonNode row, String[] columns, boolean toCsv, ObjectMapper mapper)+ throws JsonProcessingException {+ if (columns == null) {+ if (toCsv) {+ List<String> vals = new ArrayList<>();+ row.fields().forEachRemaining(e -> vals.add(csvEscape(e.getValue().asText(""))));+ return String.join(",", vals);+ }+ return mapper.writeValueAsString(row);+ }+ if (toCsv) {+ return Arrays.stream(columns)+ .map(col -> csvEscape(row.path(col).asText("")))+ .collect(Collectors.joining(","));+ }+ ObjectNode projected = mapper.createObjectNode();+ for (String col : columns) {+ JsonNode v = row.path(col);+ if (!v.isMissingNode()) {+ projected.set(col, v);+ }+ }+ return mapper.writeValueAsString(projected);+ }++ // ── Bounded reading ────────────────────────────────────────────────────++ /**+ * Reads the lines of an object, split at {@code \n} with a {@code \r} before it dropped, the way+ * the object has always been split, and fails a line longer than the record limit before holding+ * more of it.+ */+ private static final class LineReader {++ private final Reader in;+ private final char[] buffer = new char[8192];+ private int position;+ private int limit;++ LineReader(Reader in) {+ this.in = in;+ }++ /** The next line, or {@code null} at the end of the object. */+ String readLine() throws IOException {+ StringBuilder line = null;+ while (true) {+ if (position == limit) {+ int read = in.read(buffer, 0, buffer.length);+ if (read < 0) {+ if (line != null && line.length() > MAX_RECORD_CHARS) {+ throw overMaxRecordSize();+ }+ return line == null ? null : line.toString();+ }+ position = 0;+ limit = read;}- } else {- if (toCsv) {- sb.append(Arrays.stream(cols)- .map(col -> csvEscape(row.path(col).asText("")))- .collect(Collectors.joining(","))).append("\n");- } else {- ObjectNode projected = mapper.createObjectNode();- for (String col : cols) {- JsonNode v = row.path(col);- if (!v.isMissingNode()) projected.set(col, v);+ int start = position;+ while (position < limit && buffer[position] != '\n') {+ position++;+ }+ if (line == null) {+ line = new StringBuilder();+ }+ line.append(buffer, start, position - start);+ // One character over the limit can still be the \r of a \r\n.+ if (line.length() > MAX_RECORD_CHARS + 1) {+ throw overMaxRecordSize();+ }+ if (position < limit) {+ position++;+ int length = line.length();+ if (length > 0 && line.charAt(length - 1) == '\r') {+ line.setLength(length - 1);}- try { sb.append(mapper.writeValueAsString(projected)).append("\n"); } catch (Exception ignored) {}+ if (line.length() > MAX_RECORD_CHARS) {+ throw overMaxRecordSize();+ }+ return line.toString();+ }+ }+ }+ }++ /**+ * Counts what a JSON parser reads towards the array element it is on, so an element cannot grow+ * past the record limit, by more than the parser's read-ahead, before it is rejected. Everything+ * read inside an element counts. Between elements only what is not whitespace counts: whitespace+ * there is skipped without being held, while anything else is the next element, which a parser+ * can read whole before it is handed over, as it does a number.+ */+ private static final class ElementLimitedReader extends FilterReader {++ private boolean inElement;+ private long counted;++ ElementLimitedReader(Reader in) {+ super(in);+ }++ void startElement() {+ inElement = true;+ }++ void endElement() {+ inElement = false;+ counted = 0;+ }++ boolean overRecordLimit() {+ return counted > MAX_RECORD_CHARS;+ }++ @Override+ public int read() throws IOException {+ int c = super.read();+ if (c >= 0) {+ count((char) c);+ }+ return c;+ }++ @Override+ public int read(char[] buffer, int offset, int length) throws IOException {+ int read = super.read(buffer, offset, length);+ for (int i = offset; i < offset + read; i++) {+ count(buffer[i]);+ }+ return read;+ }++ private void count(char c) {+ if (inElement || !Character.isWhitespace(c)) {+ counted++;+ if (counted > MAX_RECORD_CHARS + PARSER_READ_AHEAD) {+ throw overMaxRecordSize();}}}- return sb.toString();}// ── Shared helpers ─────────────────────────────────────────────────────diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.javaindex 3156b7e843..4f26e654a0 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3SelectService.java@@ -1,14 +1,24 @@package io.github.hectorvent.floci.services.s3;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.core.common.AwsEventStreamEncoder;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.XmlBuilder;import io.github.hectorvent.floci.core.common.XmlParser;import io.github.hectorvent.floci.services.floci.duck.FlociDuckClient;import io.github.hectorvent.floci.services.s3.model.S3Object;-import com.fasterxml.jackson.databind.ObjectMapper;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.io.BufferedReader;+import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;+import java.io.FilterInputStream;+import java.io.IOException;+import java.io.InputStream;+import java.io.InputStreamReader;+import java.io.OutputStream;import java.nio.charset.StandardCharsets;import java.util.LinkedHashMap;import java.util.List;@@ -19,9 +29,14 @@ import java.util.regex.Pattern;@ApplicationScopedpublic class S3SelectService {+ private static final Logger LOG = Logger.getLogger(S3SelectService.class);+private static final Pattern S3OBJECT_PATTERN =Pattern.compile("\\bS3Object\\b", Pattern.CASE_INSENSITIVE);+ // Records are written out in events of about this size, so the response never holds them all.+ private static final int RECORDS_EVENT_SIZE = 64 * 1024;+private final ObjectMapper objectMapper;private final FlociDuckClient duckClient;@@ -31,34 +46,92 @@ public class S3SelectService {this.duckClient = duckClient;}- public byte[] select(S3Object object, String requestXml) {- String expression = XmlParser.extractFirst(requestXml, "Expression", "");- String fileHeaderInfo = XmlParser.extractFirst(requestXml, "FileHeaderInfo", "NONE");- String inputType = detectType(requestXml, "InputSerialization");- String outputFormat = detectType(requestXml, "OutputSerialization");- if (outputFormat == null) outputFormat = "CSV";-- byte[] rawData = object.getData();- if (rawData == null) return new byte[0];- long bytesScanned = rawData.length;-- String result;- if (isParquet(object)) {- result = selectParquet(object, expression, outputFormat);- } else if (duckClient.isAvailable() && canUseDuck(inputType, fileHeaderInfo)) {- result = selectViaDuck(object, expression, inputType, outputFormat);- } else if ("CSV".equals(inputType)) {- String content = new String(rawData, StandardCharsets.UTF_8);- result = S3SelectEvaluator.evaluateCsv(content, expression, fileHeaderInfo, outputFormat);- } else if ("JSON".equals(inputType)) {- String content = new String(rawData, StandardCharsets.UTF_8);- result = S3SelectEvaluator.evaluateJson(content, expression, objectMapper, outputFormat);- } else {- result = new String(rawData, StandardCharsets.UTF_8);+ /** A SelectObjectContent response, written as an AWS event stream. */+ @FunctionalInterface+ public interface EventStream {+ void writeTo(OutputStream out) throws IOException;+ }++ /**+ * Answers a SelectObjectContent request over {@code read}, an object and an open stream over the+ * body that matches it, and takes over closing {@code read}. A query floci-duck runs (Parquet, and+ * JSON or CSV with a header row while it is available) reads the object itself, so it is answered+ * here. The others are evaluated as the returned stream is written, a record at a time, so the+ * object is never held in memory. A request that cannot be answered fails here, before any of the+ * response is written; one that fails while it is written, such as on a record over S3 Select's+ * limit, ends the stream with a request-level error message after the records selected so far.+ */+ public EventStream select(S3Service.ObjectRead read, String requestXml) {+ try {+ S3Object object = read.object();+ String expression = XmlParser.extractFirst(requestXml, "Expression", "");+ String fileHeaderInfo = XmlParser.extractFirst(requestXml, "FileHeaderInfo", "NONE");+ String inputType = detectType(requestXml, "InputSerialization");+ String detectedOutput = detectType(requestXml, "OutputSerialization");+ String outputFormat = detectedOutput != null ? detectedOutput : "CSV";++ if (isParquet(object)) {+ return answered(read, selectParquet(object, expression, outputFormat), object.getSize());+ }+ if (duckClient.isAvailable() && canUseDuck(inputType, fileHeaderInfo)) {+ return answered(read, selectViaDuck(object, expression, inputType, outputFormat), object.getSize());+ }+ boolean evaluated = "CSV".equals(inputType) || "JSON".equals(inputType);+ S3SelectEvaluator.Query query = evaluated ? S3SelectEvaluator.parse(expression) : null;+ return out -> {+ try (S3Service.ObjectRead source = read) {+ CountingInputStream body = new CountingInputStream(source.body());+ RecordsWriter records = new RecordsWriter(out);+ try {+ if (query == null) {+ // Not a statement the evaluator reads: the object is returned as it is.+ records.copy(body);+ } else {+ BufferedReader reader = new BufferedReader(+ new InputStreamReader(body, StandardCharsets.UTF_8));+ if ("CSV".equals(inputType)) {+ S3SelectEvaluator.evaluateCsv(reader, query, fileHeaderInfo, outputFormat,+ records::record);+ } else {+ S3SelectEvaluator.evaluateJson(reader, query, objectMapper, outputFormat,+ records::record);+ }+ }+ } catch (AwsException e) {+ records.fail(e);+ return;+ }+ // A LIMIT can stop the query before the end of the object; what was read is what was scanned.+ records.finish(body.count());+ }+ };+ } catch (RuntimeException e) {+ closeQuietly(read);+ throw e;+ }+ }++ /** A response worked out in full already; the object's stream is not needed for it. */+ private static EventStream answered(S3Service.ObjectRead read, String result, long bytesScanned) {+ closeQuietly(read);+ try {+ ByteArrayOutputStream events = new ByteArrayOutputStream();+ RecordsWriter records = new RecordsWriter(events);+ records.copy(new ByteArrayInputStream(result.getBytes(StandardCharsets.UTF_8)));+ records.finish(bytesScanned);+ byte[] encoded = events.toByteArray();+ return out -> out.write(encoded);+ } catch (IOException e) {+ throw new IllegalStateException("Failed to encode an S3 Select response", e);}+ }- long bytesReturned = result.getBytes(StandardCharsets.UTF_8).length;- return encodeEventStream(result, bytesScanned, bytesReturned);+ private static void closeQuietly(S3Service.ObjectRead read) {+ try {+ read.close();+ } catch (IOException e) {+ LOG.debugv("Could not close an S3 Select source: {0}", e.getMessage());+ }}// ── Duck delegation ────────────────────────────────────────────────────@@ -115,16 +188,45 @@ public class S3SelectService {return null;}- private byte[] encodeEventStream(String payload, long bytesScanned, long bytesReturned) {- try {- ByteArrayOutputStream baos = new ByteArrayOutputStream();+ /**+ * Writes result records as Records events of about {@link #RECORDS_EVENT_SIZE} bytes, then the+ * Stats and End events. S3 Select may split a record across Records events; records written one+ * at a time are kept whole. A response always carries a Records event, even an empty one.+ */+ private static final class RecordsWriter {- LinkedHashMap<String, String> recordsHeaders = new LinkedHashMap<>();- recordsHeaders.put(":message-type", "event");- recordsHeaders.put(":event-type", "Records");- recordsHeaders.put(":content-type", "application/octet-stream");- baos.write(AwsEventStreamEncoder.encodeMessage(recordsHeaders, payload.getBytes(StandardCharsets.UTF_8)));+ private final OutputStream out;+ private final ByteArrayOutputStream pending = new ByteArrayOutputStream();+ private long bytesReturned;+ private boolean recordsWritten;+ RecordsWriter(OutputStream out) {+ this.out = out;+ }++ void record(String record) throws IOException {+ if (record.length() > S3SelectEvaluator.MAX_RECORD_CHARS) {+ throw S3SelectEvaluator.overMaxRecordSize();+ }+ pending.writeBytes((record + "\n").getBytes(StandardCharsets.UTF_8));+ if (pending.size() >= RECORDS_EVENT_SIZE) {+ flush();+ }+ }++ void copy(InputStream body) throws IOException {+ byte[] buffer = new byte[RECORDS_EVENT_SIZE];+ for (int read = body.readNBytes(buffer, 0, buffer.length); read > 0;+ read = body.readNBytes(buffer, 0, buffer.length)) {+ pending.write(buffer, 0, read);+ flush();+ }+ }++ void finish(long bytesScanned) throws IOException {+ if (pending.size() > 0 || !recordsWritten) {+ flush();+ }LinkedHashMap<String, String> statsHeaders = new LinkedHashMap<>();statsHeaders.put(":message-type", "event");statsHeaders.put(":event-type", "Stats");@@ -136,16 +238,78 @@ public class S3SelectService {.elem("BytesReturned", bytesReturned).end("Stats").build();- baos.write(AwsEventStreamEncoder.encodeMessage(statsHeaders, statsXml.getBytes(StandardCharsets.UTF_8)));+ out.write(AwsEventStreamEncoder.encodeMessage(statsHeaders, statsXml.getBytes(StandardCharsets.UTF_8)));LinkedHashMap<String, String> endHeaders = new LinkedHashMap<>();endHeaders.put(":message-type", "event");endHeaders.put(":event-type", "End");- baos.write(AwsEventStreamEncoder.encodeMessage(endHeaders, new byte[0]));+ out.write(AwsEventStreamEncoder.encodeMessage(endHeaders, new byte[0]));+ }++ /**+ * Ends the stream with a request-level error message, after the records selected so far, as+ * S3 Select reports an error once the response has begun.+ */+ void fail(AwsException error) throws IOException {+ if (pending.size() > 0) {+ flush();+ }+ LinkedHashMap<String, String> errorHeaders = new LinkedHashMap<>();+ errorHeaders.put(":error-code", error.getErrorCode());+ errorHeaders.put(":error-message", error.getMessage());+ errorHeaders.put(":message-type", "error");+ out.write(AwsEventStreamEncoder.encodeMessage(errorHeaders, new byte[0]));+ }++ private void flush() throws IOException {+ byte[] payload = pending.toByteArray();+ pending.reset();+ LinkedHashMap<String, String> recordsHeaders = new LinkedHashMap<>();+ recordsHeaders.put(":message-type", "event");+ recordsHeaders.put(":event-type", "Records");+ recordsHeaders.put(":content-type", "application/octet-stream");+ out.write(AwsEventStreamEncoder.encodeMessage(recordsHeaders, payload));+ bytesReturned += payload.length;+ recordsWritten = true;+ }+ }++ /** Counts the bytes read from the object, so the Stats event reports what the query scanned. */+ private static final class CountingInputStream extends FilterInputStream {++ private long count;++ CountingInputStream(InputStream in) {+ super(in);+ }++ long count() {+ return count;+ }++ @Override+ public int read() throws IOException {+ int b = super.read();+ if (b >= 0) {+ count++;+ }+ return b;+ }++ @Override+ public int read(byte[] buffer, int offset, int length) throws IOException {+ int read = super.read(buffer, offset, length);+ if (read > 0) {+ count += read;+ }+ return read;+ }- return baos.toByteArray();- } catch (Exception e) {- return payload.getBytes(StandardCharsets.UTF_8);+ @Override+ public long skip(long n) throws IOException {+ long skipped = super.skip(n);+ count += skipped;+ return skipped;}}}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/EventStreamMessages.java b/src/test/java/io/github/hectorvent/floci/services/s3/EventStreamMessages.javanew file mode 100644index 0000000000..823c107547--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/EventStreamMessages.java@@ -0,0 +1,75 @@+package io.github.hectorvent.floci.services.s3;++import java.io.ByteArrayInputStream;+import java.io.DataInputStream;+import java.io.IOException;+import java.io.UncheckedIOException;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.zip.CRC32;++/** Splits an AWS event stream into its messages, checking the prelude and message CRCs of each. */+final class EventStreamMessages {++ private EventStreamMessages() {+ }++ record Message(Map<String, String> headers, byte[] payload) {++ /** The event type of an event message, or the message type of any other. */+ String type() {+ return headers.getOrDefault(":event-type", headers.get(":message-type"));+ }++ String text() {+ return new String(payload, StandardCharsets.UTF_8);+ }+ }++ static List<Message> decode(byte[] stream) {+ List<Message> messages = new ArrayList<>();+ int offset = 0;+ try {+ while (offset < stream.length) {+ DataInputStream in = new DataInputStream(new ByteArrayInputStream(stream, offset, stream.length - offset));+ int totalLength = in.readInt();+ int headersLength = in.readInt();+ int preludeCrc = in.readInt();+ requireCrc(stream, offset, 8, preludeCrc, "prelude");+ Map<String, String> headers = new LinkedHashMap<>();+ int headersRead = 0;+ while (headersRead < headersLength) {+ int nameLength = in.readUnsignedByte();+ String name = new String(in.readNBytes(nameLength), StandardCharsets.UTF_8);+ int valueType = in.readUnsignedByte();+ if (valueType != 7) {+ throw new IllegalStateException("header " + name + " is not a string: type " + valueType);+ }+ int valueLength = in.readUnsignedShort();+ headers.put(name, new String(in.readNBytes(valueLength), StandardCharsets.UTF_8));+ headersRead += 1 + nameLength + 1 + 2 + valueLength;+ }+ byte[] payload = in.readNBytes(totalLength - 12 - headersLength - 4);+ int messageCrc = in.readInt();+ requireCrc(stream, offset, totalLength - 4, messageCrc, "message");+ messages.add(new Message(headers, payload));+ offset += totalLength;+ }+ } catch (IOException e) {+ throw new UncheckedIOException("Truncated event stream at byte " + offset, e);+ }+ return messages;+ }++ private static void requireCrc(byte[] stream, int offset, int length, int expected, String part) {+ CRC32 crc = new CRC32();+ crc.update(Arrays.copyOfRange(stream, offset, offset + length));+ if ((int) crc.getValue() != expected) {+ throw new IllegalStateException("Bad " + part + " CRC in the message at byte " + offset);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.javaindex cdf7d082f3..ff179e94e1 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectIntegrationTest.java@@ -1,11 +1,16 @@package io.github.hectorvent.floci.services.s3;+import io.github.hectorvent.floci.services.s3.EventStreamMessages.Message;import io.quarkus.test.junit.QuarkusTest;import org.junit.jupiter.api.Test;+import java.util.List;+import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.not;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;@QuarkusTestclass S3SelectIntegrationTest {@@ -73,6 +78,44 @@ class S3SelectIntegrationTest {private static final String CSV_IGNORE = "<CSV><FileHeaderInfo>IGNORE</FileHeaderInfo></CSV>";private static final String JSON_IN = "<JSON/>";+ // ── Streamed responses ────────────────────────────────────────────────++ @Test+ void select_largeResultArrivesAsSeveralRecordsEvents() {+ String bucket = "sel-streamed-result";+ StringBuilder csv = new StringBuilder();+ for (int i = 0; i < 20_000; i++) {+ csv.append("row").append(i).append(',').append(i).append('\n');+ }+ createBucketAndPut(bucket, "big.csv", csv.toString());++ byte[] response = select(bucket, "big.csv", "SELECT * FROM S3Object", CSV_NONE, CSV_OUT)+ .contentType("application/octet-stream")+ .extract().asByteArray();++ List<Message> messages = EventStreamMessages.decode(response);+ List<Message> records = messages.stream().filter(message -> "Records".equals(message.type())).toList();+ assertTrue(records.size() > 1, "one Records event for a result of " + csv.length() + " bytes");+ assertEquals(csv.toString(), String.join("", records.stream().map(Message::text).toList()));+ assertEquals(List.of("Stats", "End"), messages.subList(messages.size() - 2, messages.size()).stream()+ .map(Message::type).toList());+ assertTrue(messages.get(messages.size() - 2).text().contains("<BytesReturned>" + csv.length() + "</BytesReturned>"));+ }++ @Test+ void select_recordOverTheLimitEndsTheStreamWithAnErrorMessage() {+ String bucket = "sel-record-limit";+ createBucketAndPut(bucket, "wide.csv", "a,1\n" + "x".repeat(S3SelectEvaluator.MAX_RECORD_CHARS + 1) + "\nb,2\n");++ byte[] response = select(bucket, "wide.csv", "SELECT * FROM S3Object", CSV_NONE, CSV_OUT)+ .extract().asByteArray();++ List<Message> messages = EventStreamMessages.decode(response);+ assertEquals(List.of("Records", "error"), messages.stream().map(Message::type).toList());+ assertEquals("a,1\n", messages.get(0).text());+ assertEquals("OverMaxRecordSize", messages.get(1).headers().get(":error-code"));+ }+// ── Existing tests (preserved) ────────────────────────────────────────@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectStreamingTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectStreamingTest.javanew file mode 100644index 0000000000..bc0dd6e536--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3SelectStreamingTest.java@@ -0,0 +1,367 @@+package io.github.hectorvent.floci.services.s3;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.floci.duck.FlociDuckClient;+import io.github.hectorvent.floci.services.s3.EventStreamMessages.Message;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.nio.charset.StandardCharsets;+import java.time.Duration;+import java.util.List;+import java.util.function.LongFunction;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * SelectObjectContent over CSV and JSON read a record at a time, with the result written as it is+ * produced, so neither the object nor the result is held in memory.+ */+class S3SelectStreamingTest {++ private static final String CSV_USE = "<CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV>";+ private static final String CSV_NONE = "<CSV><FileHeaderInfo>NONE</FileHeaderInfo></CSV>";+ private static final String JSON_IN = "<JSON/>";+ private static final String CSV_OUT = "<CSV/>";++ private S3SelectService service;++ @BeforeEach+ void setUp() {+ FlociDuckClient duck = mock(FlociDuckClient.class);+ when(duck.isAvailable()).thenReturn(false);+ service = new S3SelectService(new ObjectMapper(), duck);+ }++ @Test+ void aCsvSelectStopsReadingAnEndlessObjectAtItsLimit() {+ InputStream body = endless(i -> i == 0 ? "name,age\n" : "name" + i + "," + i + "\n");++ List<Message> events = assertTimeoutPreemptively(Duration.ofSeconds(30),+ () -> select(body, "SELECT name FROM S3Object WHERE age > 5 LIMIT 3", CSV_USE, CSV_OUT));++ assertEquals("name6\nname7\nname8\n", records(events));+ }++ @Test+ void aJsonLinesSelectStopsReadingAnEndlessObjectAtItsLimit() {+ InputStream body = endless(i -> "{\"id\":" + i + "}\n");++ List<Message> events = assertTimeoutPreemptively(Duration.ofSeconds(30),+ () -> select(body, "SELECT id FROM S3Object WHERE id >= 10 LIMIT 2", JSON_IN, CSV_OUT));++ assertEquals("10\n11\n", records(events));+ }++ @Test+ void aJsonArrayIsReadAnElementAtATime() {+ InputStream body = endless(i -> (i == 0 ? "[" : ",") + "{\"id\":" + i + "}");++ List<Message> events = assertTimeoutPreemptively(Duration.ofSeconds(30),+ () -> select(body, "SELECT * FROM S3Object LIMIT 2", JSON_IN, "<JSON/>"));++ assertEquals("{\"id\":0}\n{\"id\":1}\n", records(events));+ }++ @Test+ void aMalformedJsonArrayReturnsTheRecordsBeforeIt() throws IOException {+ List<Message> events = select(bytes("[{\"id\":1},{\"id\":2},{\"id\":"), "SELECT id FROM S3Object",+ JSON_IN, CSV_OUT);++ assertEquals("1\n2\n", records(events));+ }++ @Test+ void aLargeResultIsWrittenAsSeveralRecordsEventsOfWholeRecords() throws IOException {+ StringBuilder csv = new StringBuilder();+ for (int i = 0; i < 20_000; i++) {+ csv.append("row").append(i).append(',').append(i).append('\n');+ }++ List<Message> events = select(bytes(csv.toString()), "SELECT * FROM S3Object", CSV_NONE, CSV_OUT);++ List<Message> recordEvents = events.stream().filter(event -> "Records".equals(event.type())).toList();+ assertTrue(recordEvents.size() > 1, "a result of " + csv.length() + " bytes in one event");+ for (Message event : recordEvents) {+ assertTrue(new String(event.payload(), StandardCharsets.UTF_8).endsWith("\n"), "a record split across events");+ }+ assertEquals(csv.toString(), records(events));+ assertEquals(List.of("Stats", "End"), events.subList(events.size() - 2, events.size()).stream()+ .map(Message::type).toList());+ String stats = new String(events.get(events.size() - 2).payload(), StandardCharsets.UTF_8);+ assertTrue(stats.contains("<BytesReturned>" + csv.length() + "</BytesReturned>"), stats);+ assertTrue(stats.contains("<BytesScanned>" + csv.length() + "</BytesScanned>"), stats);+ }++ @Test+ void aResultWithNoRecordsStillHasARecordsEvent() throws IOException {+ List<Message> events = select(bytes("Alice,30\nBob,25\n"), "SELECT * FROM S3Object WHERE _2 > 99",+ CSV_NONE, CSV_OUT);++ assertEquals(List.of("Records", "Stats", "End"), events.stream().map(Message::type).toList());+ assertEquals(0, events.get(0).payload().length);+ }++ @Test+ void aStatementTheEvaluatorDoesNotReadReturnsTheObjectAsItIs() throws IOException {+ byte[] object = new byte[200_000];+ for (int i = 0; i < object.length; i++) {+ object[i] = (byte) ('a' + i % 26);+ }++ List<Message> events = select(object, "DESCRIBE S3Object", CSV_NONE, CSV_OUT);++ ByteArrayOutputStream returned = new ByteArrayOutputStream();+ events.stream().filter(event -> "Records".equals(event.type()))+ .forEach(event -> returned.writeBytes(event.payload()));+ assertArrayEquals(object, returned.toByteArray());+ }++ @Test+ void aStatementThatCannotBeEvaluatedFailsBeforeTheObjectIsReadAndClosesIt() {+ TrackedStream body = new TrackedStream(bytes("Alice,30\n"));++ AwsException error = assertThrows(AwsException.class, () -> service.select(read(body, 9),+ request("SELECT * FROM S3Object s WHERE CAST(s._2 AS INTEGER) > 25", CSV_NONE, CSV_OUT)));++ assertEquals("ExternalEvalException", error.getErrorCode());+ assertFalse(body.read, "the object was read");+ assertTrue(body.closed, "the object was left open");+ }++ @Test+ void theObjectIsClosedOnceTheResponseIsWritten() throws IOException {+ TrackedStream body = new TrackedStream(bytes("Alice,30\n"));++ S3SelectService.EventStream events = service.select(read(body, 9),+ request("SELECT * FROM S3Object", CSV_NONE, CSV_OUT));+ assertFalse(body.closed, "closed before the response was written");+ events.writeTo(new ByteArrayOutputStream());++ assertTrue(body.closed);+ }++ @Test+ void aLimitThatStopsEarlyReportsTheBytesItRead() throws IOException {+ StringBuilder csv = new StringBuilder();+ for (int i = 0; i < 100_000; i++) {+ csv.append("row").append(i).append(',').append(i).append('\n');+ }++ List<Message> events = select(bytes(csv.toString()), "SELECT * FROM S3Object LIMIT 1", CSV_NONE, CSV_OUT);++ assertEquals("row0,0\n", records(events));+ long scanned = stat(events, "BytesScanned");+ assertTrue(scanned > 0 && scanned < csv.length(), "scanned " + scanned + " of " + csv.length());+ assertEquals(scanned, stat(events, "BytesProcessed"));+ }++ @Test+ void aLineOfExactlyTheRecordLimitIsARecord() throws IOException {+ String line = "x".repeat(S3SelectEvaluator.MAX_RECORD_CHARS);++ List<Message> events = select(bytes(line + "\r\nend\n"), "SELECT * FROM S3Object", CSV_NONE, CSV_OUT);++ assertEquals(line + "\nend\n", records(events));+ }++ @Test+ void aCsvLineOverTheRecordLimitEndsTheStreamWithOverMaxRecordSize() throws IOException {+ String tooLong = "x".repeat(S3SelectEvaluator.MAX_RECORD_CHARS + 1);++ List<Message> events = select(bytes("a,1\n" + tooLong + "\nb,2\n"), "SELECT * FROM S3Object",+ CSV_NONE, CSV_OUT);++ assertOverMaxRecordSizeAfter("a,1\n", events);+ }++ @Test+ void anEndlessLineEndsTheStreamWithoutBeingHeld() {+ List<Message> events = assertTimeoutPreemptively(Duration.ofSeconds(30),+ () -> select(endless(i -> "xxxxxxxxxxxxxxxx"), "SELECT * FROM S3Object", CSV_NONE, CSV_OUT));++ assertOverMaxRecordSizeAfter("", events);+ }++ @Test+ void aJsonArrayElementOverTheRecordLimitEndsTheStream() throws IOException {+ String element = "{\"id\":\"" + "x".repeat(S3SelectEvaluator.MAX_RECORD_CHARS) + "\"}";++ List<Message> events = select(bytes("[{\"id\":1}," + element + ",{\"id\":3}]"),+ "SELECT id FROM S3Object", JSON_IN, CSV_OUT);++ assertOverMaxRecordSizeAfter("1\n", events);+ }++ @Test+ void whitespaceBetweenJsonArrayElementsIsNotPartOfARecord() throws IOException {+ String gap = " \n".repeat(S3SelectEvaluator.MAX_RECORD_CHARS);++ List<Message> events = select(bytes("[{\"id\":1}," + gap + "{\"id\":2}" + gap + "]"),+ "SELECT id FROM S3Object", JSON_IN, CSV_OUT);++ assertEquals("1\n2\n", records(events));+ assertEquals("End", events.get(events.size() - 1).type());+ }++ @Test+ void aNumericJsonArrayElementOverTheRecordLimitEndsTheStream() throws IOException {+ // The parser reads a number whole and then gives up on its own length limit. The element is+ // counted only to within what the parser has buffered ahead, so this one is over the record+ // limit by more than that.+ String number = "9".repeat(S3SelectEvaluator.MAX_RECORD_CHARS + 32 * 1024);++ List<Message> events = select(bytes("[1," + number + ",3]"), "SELECT * FROM S3Object", JSON_IN, "<JSON/>");++ assertOverMaxRecordSizeAfter("1\n", events);+ }++ @Test+ void anEndlessNumericJsonArrayElementEndsTheStreamWithoutBeingHeld() {+ InputStream body = endless(i -> i == 0 ? "[1,9" : "9999999999999999");++ List<Message> events = assertTimeoutPreemptively(Duration.ofSeconds(30),+ () -> select(body, "SELECT * FROM S3Object", JSON_IN, "<JSON/>"));++ assertOverMaxRecordSizeAfter("1\n", events);+ }++ @Test+ void anEndlessJsonArrayElementEndsTheStreamWithoutBeingHeld() {+ InputStream body = endless(i -> i == 0 ? "[{\"id\":\"" : "xxxxxxxxxxxxxxxx");++ List<Message> events = assertTimeoutPreemptively(Duration.ofSeconds(30),+ () -> select(body, "SELECT * FROM S3Object", JSON_IN, "<JSON/>"));++ assertOverMaxRecordSizeAfter("", events);+ }++ @Test+ void aResultRecordOverTheRecordLimitEndsTheStream() throws IOException {+ // Each input line is under the limit; the JSON record made of the header and the value is not.+ String column = "c".repeat(600_000);+ String value = "v".repeat(500_000);++ List<Message> events = select(bytes(column + "\n" + value + "\n"), "SELECT * FROM S3Object",+ CSV_USE, "<JSON/>");++ assertOverMaxRecordSizeAfter("", events);+ }++ private List<Message> select(byte[] object, String expression, String input, String output) throws IOException {+ return select(new ByteArrayInputStream(object), object.length, expression, input, output);+ }++ private List<Message> select(InputStream object, String expression, String input, String output)+ throws IOException {+ return select(object, Long.MAX_VALUE, expression, input, output);+ }++ private List<Message> select(InputStream object, long size, String expression, String input, String output)+ throws IOException {+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ service.select(read(object, size), request(expression, input, output)).writeTo(out);+ return EventStreamMessages.decode(out.toByteArray());+ }++ private static S3Service.ObjectRead read(InputStream body, long size) {+ return new S3Service.ObjectRead(new S3Object("bucket", "data", size, "text/plain", "\"etag\""), body);+ }++ private static String request(String expression, String input, String output) {+ return """+ <SelectObjectContentRequest>+ <Expression>%s</Expression>+ <ExpressionType>SQL</ExpressionType>+ <InputSerialization>%s</InputSerialization>+ <OutputSerialization>%s</OutputSerialization>+ </SelectObjectContentRequest>+ """.formatted(expression.replace("&", "&").replace("<", "<"), input, output);+ }++ private static void assertOverMaxRecordSizeAfter(String records, List<Message> events) {+ Message last = events.get(events.size() - 1);+ assertEquals("error", last.type(), "the stream does not end with an error message");+ assertEquals("OverMaxRecordSize", last.headers().get(":error-code"));+ assertEquals("The length of a record in the input or result is greater than the maxCharsPerRecord limit of 1 MB.",+ last.headers().get(":error-message"));+ assertEquals(0, last.payload().length);+ assertEquals(records, records(events));+ assertTrue(events.stream().noneMatch(event -> "Stats".equals(event.type()) || "End".equals(event.type())),+ "an errored stream has no Stats or End");+ }++ private static long stat(List<Message> events, String name) {+ String stats = events.stream().filter(event -> "Stats".equals(event.type())).findFirst().orElseThrow().text();+ int start = stats.indexOf("<" + name + ">") + name.length() + 2;+ return Long.parseLong(stats.substring(start, stats.indexOf("</" + name + ">")));+ }++ private static String records(List<Message> events) {+ StringBuilder records = new StringBuilder();+ for (Message event : events) {+ if ("Records".equals(event.type())) {+ records.append(new String(event.payload(), StandardCharsets.UTF_8));+ }+ }+ return records.toString();+ }++ private static byte[] bytes(String text) {+ return text.getBytes(StandardCharsets.UTF_8);+ }++ /** {@code next.apply(0)}, {@code next.apply(1)}, ... encoded as UTF-8, without end. */+ private static InputStream endless(LongFunction<String> next) {+ return new InputStream() {+ private long index;+ private byte[] current = new byte[0];+ private int position;++ @Override+ public int read() {+ if (position == current.length) {+ current = next.apply(index++).getBytes(StandardCharsets.UTF_8);+ position = 0;+ }+ return current[position++] & 0xFF;+ }+ };+ }++ /** A body that records whether it was read or closed. */+ private static final class TrackedStream extends ByteArrayInputStream {+ private boolean read;+ private boolean closed;++ TrackedStream(byte[] data) {+ super(data);+ }++ @Override+ public synchronized int read() {+ read = true;+ return super.read();+ }++ @Override+ public synchronized int read(byte[] buffer, int offset, int length) {+ read = true;+ return super.read(buffer, offset, length);+ }++ @Override+ public void close() throws IOException {+ closed = true;+ super.close();+ }+ }+}