Light, fluffy, and always free - The AWS Local Emulator alternative
feat: support parameters in Redshift Data API batches (#6475)
Vũ Thế Huy36e244f10484
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.javaindex 4d1081ab6f..ef77010880 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/RedshiftDataOperationsTest.java@@ -176,6 +176,23 @@ class RedshiftDataOperationsTest {assertThat(firstChild.records().get(0).get(0).longValue()).isEqualTo(11L);}+ @Test+ @DisplayName("batch parameters are shared across statements and preserved in metadata")+ void batchBindsSharedParameters() throws Exception {+ run("CREATE TABLE compat_batch_parameters (id integer, payload varchar(100))");+ SqlParameter id = SqlParameter.builder().name("id").value("42").build();+ SqlParameter payload = SqlParameter.builder().name("payload").value("O'Reilly :literal").build();+ String batch = data.batchExecuteStatement(request -> request.clusterIdentifier(clusterId).database(DATABASE)+ .dbUser(USERNAME).parameters(id, payload)+ .sqls("INSERT INTO compat_batch_parameters VALUES (:id, :payload)",+ "SELECT payload FROM compat_batch_parameters WHERE id = :id OR id = :id")).id();+ awaitFinished(batch);++ assertThat(describe(batch).queryParameters()).containsExactlyInAnyOrder(id, payload);+ assertThat(data.getStatementResult(request -> request.id(batch)).records().get(0).get(0).stringValue())+ .isEqualTo("O'Reilly :literal");+ }+@Test@DisplayName("result paging returns every row once through SDK next tokens")void resultPagingRoundTripsSdkNextToken() throws Exception {diff --git a/docs/services/redshift-data.md b/docs/services/redshift-data.mdindex 4628485dbb..e1233f11a7 100644--- a/docs/services/redshift-data.md+++ b/docs/services/redshift-data.md@@ -116,6 +116,7 @@ for the idle timeout, 24-hour lifetime and single-query constraints.- **`GetStatementResult` and `GetStatementResultV2`** return the typed `Records` shape. `GetStatementResultV2` also returns `ResultFormat`, and when the statement was run with `ResultFormat=CSV` it returns `Records` as `CSVRecords` strings. A statement that produced no result set (an `INSERT`, `UPDATE`, `DELETE`, or DDL statement) returns `ValidationException` with the message `Statement has no result set`.- **Paging.** The page size is 1000 rows. `NextToken` is an opaque base64 row offset; there is no server-side cursor.- **`SqlParameters` / `Parameters`** are bound into a `PreparedStatement`. Named `:placeholder` markers are rewritten to positional JDBC bind parameters. Colons inside string literals, quoted identifiers, comments, PostgreSQL `::` casts, and dollar-quoted strings are left untouched. Redshift Data API parameter values are always strings on the wire; PostgreSQL coerces each bound value to the column type.+- **Batch `Parameters`** are shared by every entry in `Sqls`. Each statement can use a subset of the supplied parameters, and every supplied parameter must be referenced by at least one statement. Missing, unused, duplicate or malformed parameters are rejected with `ValidationException` before executing the batch. Parameter names can contain letters, digits and underscores, including names consisting only of digits. Values must be non-empty strings. `DescribeStatement` and `ListStatements` return the original batch parameters in `QueryParameters`; SQL text retains its named placeholders.- **`CancelStatement`** returns `{ "Status": true }` when cancellation is accepted for queued or running work. Running transactional SQL receives JDBC cancellation and rolls back before reaching `ABORTED`; cancelling a batch rolls back its earlier writes. Cancelling a statement that is already terminal returns `ValidationException` ("Could not cancel a query that is already in ABORTED state with ID: <id>"), as AWS does. Repeated cancellation and statements that already own commit return `false` in Floci. Commands requiring auto-commit, including Spectrum external DDL and `VACUUM`, own completion before performing irreversible work and reject cancellation during that work. An unknown statement id returns `ResourceNotFoundException`.- **`WithEvent=true`** publishes one terminal status event per parent statement to the default EventBridge bus in the captured account and region. Batch children do not emit events. False or omitted `WithEvent` emits none. Delivery occurs after terminal metadata is stored, on a separate bounded pool; disabled EventBridge, rejected delivery or target failures do not change the SQL outcome. Delivery is best effort, with no durable outbox or restart replay.- **`ExecuteSql` and `BatchExecuteSql`** (the deprecated pre-2020 operations) return `ValidationException`.diff --git a/src/main/java/io/github/hectorvent/floci/core/common/SqlParameterParser.java b/src/main/java/io/github/hectorvent/floci/core/common/SqlParameterParser.javaindex 1e37b4a134..13319eb1ad 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/SqlParameterParser.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/SqlParameterParser.java@@ -35,7 +35,7 @@ public final class SqlParameterParser {*/public record Options(boolean allowBackticks, boolean backslashEscapes, boolean escapeStrings,boolean numericNames, boolean mysqlComments) {- public static final Options REDSHIFT = new Options(false, false, true, false, false);+ public static final Options REDSHIFT = new Options(false, false, true, true, false);public static final Options RDS_MYSQL = new Options(true, true, false, true, true);public static final Options RDS_POSTGRESQL = new Options(true, false, false, true, false);}diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.javaindex 1d9ce31a9b..e47580b5ba 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataService.java@@ -382,11 +382,7 @@ public class RedshiftDataService implements Resettable {stored.dbUser = target.user();stored.database = target.database();stored.resultFormat = request.path("ResultFormat").asText("JSON");- if (!parameters.isEmpty()) {- stored.queryParameters = parameters.entrySet().stream()- .map(entry -> new StoredParameter(entry.getKey(), entry.getValue()))- .toList();- }+ stored.queryParameters = storedParameters(parameters);capture(stored, target, region, request);return new PendingStatement(stored, (execution, session) -> runStatement(stored, target, sql, parameters, execution, session),@@ -792,6 +788,7 @@ public class RedshiftDataService implements Resettable {for (JsonNode n : sqlsNode) {sqls.add(n.asText());}+ Map<String, String> parameters = RedshiftDataSqlParameters.parseBatchParameters(request, sqls);ResolvedTarget resolved = resolveTarget(request, region, owner);RedshiftDataResourceResolver.DatabaseTarget target = resolved.target();@@ -806,13 +803,14 @@ public class RedshiftDataService implements Resettable {parent.database = target.database();parent.resultFormat = request.path("ResultFormat").asText("JSON");parent.subStatements = new ArrayList<>();+ parent.queryParameters = storedParameters(parameters);capture(parent, target, region, request);- return new PendingStatement(parent, (execution, session) -> runBatch(parent, sqls, target, execution, session),+ return new PendingStatement(parent, (execution, session) -> runBatch(parent, sqls, parameters, target, execution, session),target, request.deepCopy(), owner);}- private void runBatch(RedshiftDataStatementStore.StoredStatement parent, List<String> sqls,+ private void runBatch(RedshiftDataStatementStore.StoredStatement parent, List<String> sqls, Map<String, String> parameters,RedshiftDataResourceResolver.DatabaseTarget target, RedshiftDataExecution execution,RedshiftDataSessions.Reservation session) {long totalDuration = 0;@@ -849,7 +847,7 @@ public class RedshiftDataService implements Resettable {sub.updatedAt = now;try {execution.checkCancellation();- runOnConnection(sub, connection, sqls.get(n), Map.of(), target.spectrum(), execution, false);+ runOnConnection(sub, connection, sqls.get(n), parameters, target.spectrum(), execution, false);} catch (SQLException e) {sub.status = RedshiftDataStatementStore.Status.FAILED;sub.error = e.getMessage();@@ -1238,6 +1236,11 @@ public class RedshiftDataService implements Resettable {}}+ private static List<StoredParameter> storedParameters(Map<String, String> parameters) {+ return parameters.isEmpty() ? null : parameters.entrySet().stream()+ .map(entry -> new StoredParameter(entry.getKey(), entry.getValue())).toList();+ }+private static void putQueryParameters(ObjectNode target, List<StoredParameter> parameters) {if (parameters == null || parameters.isEmpty()) {return;diff --git a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSqlParameters.java b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSqlParameters.javaindex 5d0714aebe..c9a8996e67 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSqlParameters.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSqlParameters.java@@ -9,8 +9,11 @@ import java.sql.PreparedStatement;import java.sql.SQLException;import java.sql.Types;import java.util.LinkedHashMap;+import java.util.LinkedHashSet;import java.util.List;import java.util.Map;+import java.util.Set;+import java.util.regex.Pattern;/*** Translates Redshift Data API {@code SqlParameter} bindings into JDBC@@ -24,6 +27,8 @@ import java.util.Map;*/final class RedshiftDataSqlParameters {+ private static final Pattern PARAMETER_NAME = Pattern.compile("[0-9a-zA-Z_]+");+private RedshiftDataSqlParameters() {}@@ -87,6 +92,42 @@ final class RedshiftDataSqlParameters {return byName;}+ static Map<String, String> parseBatchParameters(JsonNode request, List<String> sqls) {+ JsonNode supplied = request.get("Parameters");+ if (supplied != null && !supplied.isNull()) {+ if (!supplied.isArray() || supplied.isEmpty()) {+ throw new AwsException("ValidationException", "Parameters must be a non-empty array of {name, value} objects.", 400);+ }+ for (JsonNode parameter : supplied) {+ JsonNode name = parameter.path("name");+ JsonNode value = parameter.path("value");+ if (!name.isTextual() || !PARAMETER_NAME.matcher(name.textValue()).matches()+ || !value.isTextual() || value.textValue().isEmpty()) {+ throw new AwsException("ValidationException",+ "Each parameter must have a name matching [0-9a-zA-Z_]+ and a non-empty string value.", 400);+ }+ }+ }+ Map<String, String> parameters = parseParameters(request, "Parameters");+ Set<String> referenced = new LinkedHashSet<>();+ for (String sql : sqls) {+ referenced.addAll(parse(sql).parameterOrder());+ }+ for (String name : referenced) {+ if (!parameters.containsKey(name)) {+ throw new AwsException("ValidationException",+ "SQL references parameter :" + name + " but no matching value was supplied.", 400);+ }+ }+ for (String name : parameters.keySet()) {+ if (!referenced.contains(name)) {+ throw new AwsException("ValidationException",+ "Parameter :" + name + " is not referenced by any SQL statement in the batch.", 400);+ }+ }+ return parameters;+ }+/*** Whether {@code sql} holds more than one statement, applying the same* literal / identifier / comment / dollar-quote skipping as {@link #parse}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/SqlParameterParserTest.java b/src/test/java/io/github/hectorvent/floci/core/common/SqlParameterParserTest.javaindex 93c905913c..53f4481cdf 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/SqlParameterParserTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/SqlParameterParserTest.java@@ -49,11 +49,11 @@ class SqlParameterParserTest {}@Test- void redshiftKeepsColonDigitAsLiteralText() {+ void redshiftAcceptsParameterNamesThatStartWithADigit() {ParsedSql parsed = SqlParameterParser.parse("select :1", Options.REDSHIFT);- assertEquals("select :1", parsed.sql());- assertEquals(List.of(), parsed.parameterOrder());+ assertEquals("select ?", parsed.sql());+ assertEquals(List.of("1"), parsed.parameterOrder());}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.javaindex 4bc5d4f569..d3ac1ba166 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataServiceTest.java@@ -37,6 +37,7 @@ import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;import static org.mockito.Mockito.when;class RedshiftDataServiceTest {@@ -52,6 +53,7 @@ class RedshiftDataServiceTest {private RedshiftDataService service;private SpectrumQueryPreparation preparation;private RedshiftDataExecutor executor;+ private RedshiftDataConnectionFactory factory;@BeforeEachvoid setUp() throws Exception {@@ -62,7 +64,7 @@ class RedshiftDataServiceTest {"arn:aws:redshift:us-east-1:000000000000:cluster:wh", "127.0.0.1", 5439, "dev", "admin", "x",new SpectrumSession("000000000000", "000000000000:wh", "dev", List.of(), false)));- RedshiftDataConnectionFactory factory = mock(RedshiftDataConnectionFactory.class);+ factory = mock(RedshiftDataConnectionFactory.class);when(factory.open(any())).thenAnswer(inv -> DriverManager.getConnection(jdbcUrl, "sa", ""));preparation = mock(SpectrumQueryPreparation.class);@@ -286,6 +288,84 @@ class RedshiftDataServiceTest {return r;}+ @Test+ void batchBindsSharedParametersAndReportsOriginalValues() {+ service.executeStatement(req("CREATE TABLE batch_parameters(id integer, payload varchar(100))"), REGION);+ ObjectNode request = batchReq("INSERT INTO batch_parameters VALUES (:id, :value)",+ "UPDATE batch_parameters SET payload = :updated WHERE id = :id",+ "SELECT payload FROM batch_parameters WHERE id = :id OR id = :id");+ ArrayNode parameters = request.putArray("Parameters");+ parameters.addObject().put("name", "id").put("value", "42");+ parameters.addObject().put("name", "value").put("value", "first");+ parameters.addObject().put("name", "updated").put("value", "Robert'); DROP TABLE batch_parameters; --");++ String id = service.batchExecuteStatement(request, REGION).path("Id").asText();++ ObjectNode description = service.describeStatement(idOf(id));+ assertEquals("FINISHED", description.path("Status").asText(), description.toString());+ assertEquals(parameters, description.path("QueryParameters"));+ assertEquals(request.path("Sqls").get(0).asText(), description.path("SubStatements").get(0).path("QueryString").asText());+ assertEquals("Robert'); DROP TABLE batch_parameters; --",+ service.getStatementResult(idOf(id)).path("Records").get(0).get(0).path("stringValue").asText());+ assertEquals(parameters, service.listStatements(om.createObjectNode()).path("Statements").get(0).path("QueryParameters"));+ }++ @Test+ void batchParametersRollBackEarlierWritesWhenALaterStatementFails() {+ service.executeStatement(req("CREATE TABLE batch_rollback(id integer)"), REGION);+ ObjectNode request = batchReq("INSERT INTO batch_rollback VALUES (:id)", "INSERT INTO nonexistent VALUES (:id)");+ request.putArray("Parameters").addObject().put("name", "id").put("value", "42");+ String id = service.batchExecuteStatement(request, REGION).path("Id").asText();+ assertEquals("FAILED", service.describeStatement(idOf(id)).path("Status").asText());+ String count = service.executeStatement(req("SELECT COUNT(*) FROM batch_rollback"), REGION).path("Id").asText();+ assertEquals(0, service.getStatementResult(idOf(count)).path("Records").get(0).get(0).path("longValue").asInt());+ }++ @ParameterizedTest+ @ValueSource(strings = {+ "[]", "{}", "[{}]", "[{\"name\":\"id\"}]", "[{\"name\":\"id\",\"value\":null}]",+ "[{\"name\":\"id\",\"value\":42}]", "[{\"name\":42,\"value\":\"42\"}]",+ "[{\"name\":\"id\",\"value\":\"\"}]", "[{\"name\":\"bad-name\",\"value\":\"42\"}]",+ "[{\"name\":\"id\",\"value\":\"1\"},{\"name\":\"id\",\"value\":\"2\"}]"+ })+ void invalidBatchParameterObjectsAreRejectedBeforeExecution(String json) throws Exception {+ ObjectNode request = batchReq("SELECT 1", "SELECT :id");+ request.set("Parameters", om.readTree(json));+ AwsException error = assertThrows(AwsException.class, () -> service.batchExecuteStatement(request, REGION));+ assertEquals("ValidationException", error.getErrorCode());+ verifyNoInteractions(factory);+ }++ @Test+ void missingBatchParameterIsRejectedBeforeAnyStatementRuns() {+ AwsException error = assertThrows(AwsException.class,+ () -> service.batchExecuteStatement(batchReq("SELECT 1", "SELECT :missing"), REGION));+ assertEquals("ValidationException", error.getErrorCode());+ verifyNoInteractions(factory);+ }++ @Test+ void parameterMentionedOnlyInLiteralsAndCommentsIsUnused() {+ ObjectNode request = batchReq("SELECT ':unused' /* :unused */ -- :unused\n", "SELECT $tag$:unused$tag$");+ request.putArray("Parameters").addObject().put("name", "unused").put("value", "42");+ AwsException error = assertThrows(AwsException.class, () -> service.batchExecuteStatement(request, REGION));+ assertEquals("ValidationException", error.getErrorCode());+ verifyNoInteractions(factory);+ }++ @Test+ void batchAcceptsNumericParameterNamesAndDoesNotReexecuteClientTokenRetries() {+ service.executeStatement(req("CREATE TABLE numeric_batch(id integer)"), REGION);+ ObjectNode request = batchReq("INSERT INTO numeric_batch VALUES (:1)", "SELECT id FROM numeric_batch WHERE id = :1");+ request.put("ClientToken", "numeric-batch");+ request.putArray("Parameters").addObject().put("name", "1").put("value", "42");+ String id = service.batchExecuteStatement(request, REGION).path("Id").asText();+ assertEquals("FINISHED", service.describeStatement(idOf(id)).path("Status").asText());+ assertEquals(id, service.batchExecuteStatement(request, REGION).path("Id").asText());+ String count = service.executeStatement(req("SELECT COUNT(*) FROM numeric_batch"), REGION).path("Id").asText();+ assertEquals(1, service.getStatementResult(idOf(count)).path("Records").get(0).get(0).path("longValue").asInt());+ }+@Testvoid batchExecutesInOneTransactionAndDescribeShowsSubStatements() {String id = service.batchExecuteStatement(batchReq(diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSessionIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSessionIntegrationTest.javaindex 4d9b4449d4..b8482af0f9 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSessionIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSessionIntegrationTest.java@@ -21,11 +21,13 @@ import software.amazon.awssdk.services.redshiftdata.model.DescribeStatementRespoimport software.amazon.awssdk.services.redshiftdata.model.ExecuteStatementRequest;import software.amazon.awssdk.services.redshiftdata.model.ExecuteStatementResponse;import software.amazon.awssdk.services.redshiftdata.model.ResourceNotFoundException;+import software.amazon.awssdk.services.redshiftdata.model.SqlParameter;import software.amazon.awssdk.services.redshiftdata.model.StatusString;import software.amazon.awssdk.services.redshiftdata.model.ValidationException;import java.net.URI;import java.sql.DriverManager;+import java.util.List;import java.util.UUID;import static org.junit.jupiter.api.Assertions.*;@@ -115,6 +117,41 @@ class RedshiftDataSessionIntegrationTest {finished(data.executeStatement(r -> r.waitTimeSeconds(30).sessionId(created.sessionId()).sql("SELECT 3")).id());}+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void sdkBatchBindsSharedParametersAndReturnsTheirMetadata(boolean serverless) throws Exception {+ ExecuteStatementRequest.Builder create = ExecuteStatementRequest.builder().waitTimeSeconds(30).database("dev")+ .sessionKeepAliveSeconds(60).sql("CREATE LOCAL TEMPORARY TABLE batch_params(id integer, payload varchar(100))");+ if (serverless) {+ create.workgroupName("analytics");+ } else {+ create.clusterIdentifier("warehouse").dbUser("admin");+ }+ ExecuteStatementResponse created = data.executeStatement(create.build());+ finished(created.id());+ SqlParameter id = SqlParameter.builder().name("id").value("42").build();+ SqlParameter payload = SqlParameter.builder().name("payload").value("O'Reilly :not_a_parameter").build();+ String batch = data.batchExecuteStatement(r -> r.waitTimeSeconds(30).sessionId(created.sessionId())+ .sqls("INSERT INTO batch_params VALUES (:id, :payload)", "SELECT payload FROM batch_params WHERE id = :id")+ .parameters(id, payload)).id();++ DescribeStatementResponse described = finished(batch);+ assertEquals(List.of(id, payload), described.queryParameters());+ assertEquals(2, described.subStatements().size());+ assertEquals("O'Reilly :not_a_parameter",+ data.getStatementResult(r -> r.id(batch)).records().getFirst().getFirst().stringValue());+ verify(factory).open(any());+ }++ @Test+ void sdkRejectsUnusedBatchParameterBeforeOpeningAConnection() {+ ValidationException error = assertThrows(ValidationException.class, () -> data.batchExecuteStatement(r -> r+ .clusterIdentifier("warehouse").dbUser("admin").database("dev").sqls("SELECT 1", "SELECT ':unused'")+ .parameters(SqlParameter.builder().name("unused").value("42").build())));+ assertEquals(400, error.statusCode());+ verifyNoInteractions(factory);+ }+@Testvoid sdkValidatesKeepAliveAndReplaysAClosedSessionReceipt() {assertThrows(ValidationException.class, () -> data.executeStatement(r -> r.waitTimeSeconds(30).clusterIdentifier("warehouse")diff --git a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSqlParametersTest.java b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSqlParametersTest.javaindex ef3702c39c..71fb00f805 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSqlParametersTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshiftdata/RedshiftDataSqlParametersTest.java@@ -37,6 +37,13 @@ class RedshiftDataSqlParametersTest {assertEquals(List.of("a", "a"), parsed.parameterOrder());}+ @Test+ void numericParameterNamesDoNotRewriteArraySliceBounds() {+ ParsedSql parsed = RedshiftDataSqlParameters.parse("SELECT :1::int, arr[1:2], :1, :2 FROM t");+ assertEquals("SELECT ?::int, arr[1:2], ?, ? FROM t", parsed.sql());+ assertEquals(List.of("1", "1", "2"), parsed.parameterOrder());+ }+@Testvoid bindsValuesAsStringsAndDatabaseCoerces() throws Exception {String url = "jdbc:h2:mem:" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;MODE=PostgreSQL";