Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): validate query key types (#2469)
Tom Longhursta57c6c8112cd
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbAccessPathValidationTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbAccessPathValidationTest.javaindex e205b3b1b9..14003fd7d8 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbAccessPathValidationTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbAccessPathValidationTest.java@@ -84,6 +84,16 @@ class DynamoDbAccessPathValidationTest {":status", value("open")))));}+ @Test+ void queryRejectsKeyConditionValuesWithWrongSchemaTypes() {+ assertValidationException(() -> ddb.query(request -> request+ .tableName(TABLE)+ .keyConditionExpression("pk = :pk AND sk > :sk")+ .expressionAttributeValues(Map.of(+ ":pk", value("p1"),+ ":sk", numberValue("1")))));+ }+@Testvoid acceptsTableKeyConditionsInEitherOrder() {var response = ddb.query(request -> request@@ -223,6 +233,10 @@ class DynamoDbAccessPathValidationTest {return AttributeValue.builder().s(value).build();}+ private static AttributeValue numberValue(String value) {+ return AttributeValue.builder().n(value).build();+ }+private static Condition condition(ComparisonOperator operator, String attributeValue) {return Condition.builder().comparisonOperator(operator)diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathValidator.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathValidator.javaindex 29fda87189..93cf88c621 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathValidator.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathValidator.java@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.services.dynamodb.ExpressionEvaluator.OrExpr;import io.github.hectorvent.floci.services.dynamodb.ExpressionEvaluator.PathOperand;import io.github.hectorvent.floci.services.dynamodb.ExpressionEvaluator.PlaceholderOperand;import io.github.hectorvent.floci.services.dynamodb.ExpressionEvaluator.TokenType;+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;import java.util.HashSet;@@ -23,6 +24,8 @@ import java.util.Set;final class DynamoDbAccessPathValidator {+ private static final String KEY_TYPE_MISMATCH =+ "One or more parameter values were invalid: Condition parameter type does not match schema type";private static final Set<TokenType> SORT_KEY_COMPARATORS = Set.of(TokenType.EQ, TokenType.LT, TokenType.LE, TokenType.GT, TokenType.GE);private static final Set<String> LEGACY_SORT_KEY_COMPARATORS = Set.of(@@ -30,15 +33,17 @@ final class DynamoDbAccessPathValidator {private DynamoDbAccessPathValidator() {}- static String validateQuery(DynamoDbAccessPath accessPath, JsonNode keyConditions,+ static String validateQuery(TableDefinition table, DynamoDbAccessPath accessPath, JsonNode keyConditions,String keyConditionExpression, String filterExpression,- JsonNode queryFilter, JsonNode expressionAttributeNames) {+ JsonNode queryFilter, JsonNode expressionAttributeNames,+ JsonNode expressionAttributeValues) {String partitionKeyValuePlaceholder = null;if (keyConditionExpression != null) {partitionKeyValuePlaceholder = validateKeyConditionExpression(- accessPath, keyConditionExpression, expressionAttributeNames);+ table, accessPath, keyConditionExpression,+ expressionAttributeNames, expressionAttributeValues);} else {- validateLegacyKeyConditions(accessPath, keyConditions);+ validateLegacyKeyConditions(table, accessPath, keyConditions);}validateFilterExpression(accessPath, filterExpression, expressionAttributeNames);validateLegacyQueryFilter(accessPath, queryFilter);@@ -83,8 +88,10 @@ final class DynamoDbAccessPathValidator {}}- private static String validateKeyConditionExpression(DynamoDbAccessPath accessPath,- String expression, JsonNode names) {+ private static String validateKeyConditionExpression(TableDefinition table,+ DynamoDbAccessPath accessPath,+ String expression, JsonNode names,+ JsonNode values) {Expr root = parseExpression(expression, "KeyConditionExpression");List<Expr> conditions = root instanceof AndExpr and@@ -117,6 +124,7 @@ final class DynamoDbAccessPathValidator {} else {throw new AwsException("ValidationException", "Query key condition not supported", 400);}+ validateConditionValueTypes(table, attribute, condition, values);}if (!conditionedPartitionKeys.containsAll(partitionKeySet)) {@@ -166,7 +174,39 @@ final class DynamoDbAccessPathValidator {return topLevelAttribute(path, names);}- private static void validateLegacyKeyConditions(DynamoDbAccessPath accessPath, JsonNode keyConditions) {+ private static void validateConditionValueTypes(TableDefinition table, String attribute,+ Expr condition, JsonNode values) {+ String expectedType = attributeType(table, attribute);+ for (String placeholder : conditionValuePlaceholders(condition)) {+ if (values == null || !values.has(placeholder)) {+ throw validationException("Invalid KeyConditionExpression: An expression attribute value used "+ + "in expression is not defined; attribute value: " + placeholder);+ }+ if (!isValidKeyValue(values.get(placeholder), expectedType)) {+ throw validationException(KEY_TYPE_MISMATCH);+ }+ }+ }++ private static List<String> conditionValuePlaceholders(Expr condition) {+ return switch (condition) {+ case CompareExpr compare when compare.right() instanceof PlaceholderOperand placeholder ->+ List.of(placeholder.name());+ case BetweenExpr between+ when between.low() instanceof PlaceholderOperand low+ && between.high() instanceof PlaceholderOperand high ->+ List.of(low.name(), high.name());+ case FunctionCallExpr function+ when function.args().size() == 2+ && function.args().get(1) instanceof PlaceholderOperand placeholder ->+ List.of(placeholder.name());+ default -> List.of();+ };+ }++ private static void validateLegacyKeyConditions(TableDefinition table,+ DynamoDbAccessPath accessPath,+ JsonNode keyConditions) {List<String> partitionKeys = accessPath.partitionKeyNames();for (String partitionKey : partitionKeys) {if (keyConditions == null || !keyConditions.has(partitionKey)) {@@ -190,9 +230,45 @@ final class DynamoDbAccessPathValidator {|| ("BETWEEN".equals(operator) ? values != 2 : values != 1)) {throw new AwsException("ValidationException", "Query key condition not supported", 400);}+ String expectedType = attributeType(table, attribute);+ entry.getValue().path("AttributeValueList").forEach(value -> {+ if (!isValidKeyValue(value, expectedType)) {+ throw validationException(KEY_TYPE_MISMATCH);+ }+ });});}+ private static String attributeType(TableDefinition table, String attribute) {+ List<AttributeDefinition> definitions = table.getAttributeDefinitions();+ if (definitions == null) {+ throw validationException(KEY_TYPE_MISMATCH);+ }+ return definitions.stream()+ .filter(definition -> attribute.equals(definition.getAttributeName()))+ .map(AttributeDefinition::getAttributeType)+ .findFirst()+ .orElseThrow(() -> validationException(KEY_TYPE_MISMATCH));+ }++ private static boolean isValidKeyValue(JsonNode value, String expectedType) {+ if (value == null || !value.isObject() || value.size() != 1 || !value.has(expectedType)) {+ return false;+ }+ JsonNode payload = value.get(expectedType);+ if (payload == null || !payload.isTextual() || payload.textValue().isEmpty()) {+ return false;+ }+ if ("N".equals(expectedType)) {+ DynamoDbNumberUtils.validateAndNormalize(payload.textValue());+ }+ if ("B".equals(expectedType)) {+ return payload.textValue().matches(+ "(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?");+ }+ return true;+ }+private static void validateFilterExpression(DynamoDbAccessPath accessPath,String expression, JsonNode names) {if (expression == null) {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.javaindex 93d1447074..19856565fa 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java@@ -866,8 +866,8 @@ public class DynamoDbJsonHandler {TableDefinition queryTable = dynamoDbService.describeTable(tableName, region);DynamoDbAccessPath queryAccessPath = DynamoDbAccessPath.resolve(queryTable, indexName);- DynamoDbAccessPathValidator.validateQuery(queryAccessPath, keyConditions, keyConditionExpr,- filterExpr, queryFilter, exprAttrNames);+ DynamoDbAccessPathValidator.validateQuery(queryTable, queryAccessPath, keyConditions,+ keyConditionExpr, filterExpr, queryFilter, exprAttrNames, exprAttrValues);DynamoDbAccessPathValidator.validateSelection(queryTable, queryAccessPath, select,projectionExpression, attributesToGet, exprAttrNames);diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 451941d02b..59e1dfd740 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -271,6 +271,15 @@ public class DynamoDbService {if (lsis != null) {lsis.forEach(l -> l.getKeySchema().forEach(k -> referencedAttrs.add(k.getAttributeName())));}+ Set<String> definedAttrs = attributeDefinitions == null+ ? Set.of()+ : attributeDefinitions.stream()+ .map(AttributeDefinition::getAttributeName)+ .collect(java.util.stream.Collectors.toSet());+ if (!definedAttrs.containsAll(referencedAttrs)) {+ throw new AwsException("ValidationException",+ "Invalid KeySchema: Some index key attribute have no definition", 400);+ }if (attributeDefinitions != null) {for (AttributeDefinition ad : attributeDefinitions) {if (!referencedAttrs.contains(ad.getAttributeName())) {@@ -755,7 +764,8 @@ public class DynamoDbService {DynamoDbAccessPath accessPath = DynamoDbAccessPath.resolve(table, indexName);String partitionKeyValuePlaceholder = DynamoDbAccessPathValidator.validateQuery(- accessPath, keyConditions, keyConditionExpression, filterExpression, null, exprAttrNames);+ table, accessPath, keyConditions, keyConditionExpression, filterExpression,+ null, exprAttrNames, expressionAttrValues);String pkName = accessPath.partitionKeyName();List<String> pkNames = accessPath.partitionKeyNames();String skName = accessPath.sortKeyName();diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathIntegrationTest.javaindex 7faf13dbbf..7c442f6012 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathIntegrationTest.java@@ -142,6 +142,45 @@ class DynamoDbAccessPathIntegrationTest {@Test@Order(5)+ void queryRejectsKeyConditionValuesWithWrongSchemaTypes() {+ request("DynamoDB_20120810.Query", """+ {+ "TableName":"%s",+ "KeyConditionExpression":"pk = :pk AND sk > :sk",+ "ExpressionAttributeValues":{+ ":pk":{"S":"p1"},+ ":sk":{"N":"1"}+ }+ }+ """.formatted(TABLE))+ .statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("One or more parameter values were invalid: "+ + "Condition parameter type does not match schema type"));++ request("DynamoDB_20120810.Query", """+ {+ "TableName":"%s",+ "KeyConditionExpression":"pk = :pk",+ "ExpressionAttributeValues":{":pk":{"S":null}}+ }+ """.formatted(TABLE))+ .statusCode(400)+ .body("__type", equalTo("ValidationException"));++ request("DynamoDB_20120810.Query", """+ {+ "TableName":"%s",+ "KeyConditionExpression":"pk = :pk",+ "ExpressionAttributeValues":{":pk":{"S":"p1","N":"1"}}+ }+ """.formatted(TABLE))+ .statusCode(400)+ .body("__type", equalTo("ValidationException"));+ }++ @Test+ @Order(6)void queryRejectsSelectedKeyInFilterExpression() {request("DynamoDB_20120810.Query", """{@@ -163,7 +202,7 @@ class DynamoDbAccessPathIntegrationTest {}@Test- @Order(6)+ @Order(7)void queryAndScanRejectNonProjectedGsiAttribute() {String query = """{@@ -191,7 +230,7 @@ class DynamoDbAccessPathIntegrationTest {}@Test- @Order(7)+ @Order(8)void acceptsProjectedGsiAttributesAndLsiTableFetch() {request("DynamoDB_20120810.Query", """{@@ -230,7 +269,7 @@ class DynamoDbAccessPathIntegrationTest {}@Test- @Order(8)+ @Order(9)void gsiStillRejectsConsistentReads() {request("DynamoDB_20120810.Query", """{diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathValidatorTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathValidatorTest.javaindex c9d6467353..dd0395f308 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathValidatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccessPathValidatorTest.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;@@ -32,6 +33,14 @@ class DynamoDbAccessPathValidatorTest {table.setKeySchema(List.of(new KeySchemaElement("pk", "HASH"),new KeySchemaElement("sk", "RANGE")));+ table.setAttributeDefinitions(List.of(+ new AttributeDefinition("pk", "S"),+ new AttributeDefinition("sk", "S"),+ new AttributeDefinition("status", "S"),+ new AttributeDefinition("createdAt", "S"),+ new AttributeDefinition("alternate", "S"),+ new AttributeDefinition("tenantId", "S"),+ new AttributeDefinition("region", "S")));table.setGlobalSecondaryIndexes(List.of(new GlobalSecondaryIndex("status-index",@@ -84,13 +93,13 @@ class DynamoDbAccessPathValidatorTest {valid.set("tenantId", legacyCondition("EQ", attributeValues("acme")));valid.set("region", legacyCondition("EQ", attributeValues("us")));assertDoesNotThrow(() -> DynamoDbAccessPathValidator.validateQuery(- compositePkGsiPath, valid, null, null, null, null));+ table, compositePkGsiPath, valid, null, null, null, null, null));ObjectNode missingRegion = mapper.createObjectNode();missingRegion.set("tenantId", legacyCondition("EQ", attributeValues("acme")));AwsException error = assertThrows(AwsException.class,() -> DynamoDbAccessPathValidator.validateQuery(- compositePkGsiPath, missingRegion, null, null, null, null));+ table, compositePkGsiPath, missingRegion, null, null, null, null, null));assertEquals("Query condition missed key schema element: region", error.getMessage());}@@ -117,8 +126,8 @@ class DynamoDbAccessPathValidatorTest {names.put("#status", "status");assertEquals(":status", DynamoDbAccessPathValidator.validateQuery(- gsiPath, null, "#status = :status AND createdAt BETWEEN :start AND :end",- null, null, names));+ table, gsiPath, null, "#status = :status AND createdAt BETWEEN :start AND :end",+ null, null, names, expressionValues(":status", ":start", ":end")));}@Test@@ -161,7 +170,8 @@ class DynamoDbAccessPathValidatorTest {AwsException filterError = assertThrows(AwsException.class,() -> DynamoDbAccessPathValidator.validateQuery(- tablePath, null, "pk = :pk", "status =", null, null));+ table, tablePath, null, "pk = :pk", "status =", null, null,+ expressionValues(":pk")));assertEquals("Invalid FilterExpression: Syntax error", filterError.getMessage());}@@ -171,28 +181,92 @@ class DynamoDbAccessPathValidatorTest {valid.set("pk", legacyCondition("EQ", attributeValues("p1")));valid.set("sk", legacyCondition("BETWEEN", attributeValues("a", "z")));assertDoesNotThrow(() -> DynamoDbAccessPathValidator.validateQuery(- tablePath, valid, null, null, null, null));+ table, tablePath, valid, null, null, null, null, null));ObjectNode nonKey = valid.deepCopy();nonKey.set("status", legacyCondition("EQ", attributeValues("open")));assertThrows(AwsException.class, () -> DynamoDbAccessPathValidator.validateQuery(- tablePath, nonKey, null, null, null, null));+ table, tablePath, nonKey, null, null, null, null, null));ObjectNode missingPartitionKey = mapper.createObjectNode();missingPartitionKey.set("sk", legacyCondition("EQ", attributeValues("a")));assertThrows(AwsException.class, () -> DynamoDbAccessPathValidator.validateQuery(- tablePath, missingPartitionKey, null, null, null, null));+ table, tablePath, missingPartitionKey, null, null, null, null, null));}@Testvoid rejectsSelectedKeysInQueryFilters() {assertThrows(AwsException.class, () -> DynamoDbAccessPathValidator.validateQuery(- gsiPath, null, "status = :status", "createdAt > :cutoff", null, null));+ table, gsiPath, null, "status = :status", "createdAt > :cutoff", null, null,+ expressionValues(":status", ":cutoff")));ObjectNode queryFilter = mapper.createObjectNode();queryFilter.set("status", legacyCondition("EQ", attributeValues("open")));assertThrows(AwsException.class, () -> DynamoDbAccessPathValidator.validateQuery(- gsiPath, null, "status = :status", null, queryFilter, null));+ table, gsiPath, null, "status = :status", null, queryFilter, null,+ expressionValues(":status")));+ }++ @Test+ void rejectsKeyConditionValuesWithWrongSchemaTypes() {+ ObjectNode values = expressionValues(":pk", ":sk");+ values.set(":pk", numberValue("1"));++ AwsException expressionError = assertThrows(AwsException.class,+ () -> DynamoDbAccessPathValidator.validateQuery(+ table, tablePath, null, "pk = :pk AND sk > :sk",+ null, null, null, values));+ assertEquals("One or more parameter values were invalid: "+ + "Condition parameter type does not match schema type", expressionError.getMessage());++ ObjectNode legacy = mapper.createObjectNode();+ legacy.set("pk", legacyCondition("EQ", attributeValues("p1")));+ legacy.set("sk", legacyCondition("EQ", mapper.createArrayNode().add(numberValue("1"))));+ assertThrows(AwsException.class, () -> DynamoDbAccessPathValidator.validateQuery(+ table, tablePath, legacy, null, null, null, null, null));+ }++ @Test+ void rejectsMalformedKeyConditionValues() {+ ObjectNode nullString = mapper.createObjectNode();+ nullString.putNull("S");+ ObjectNode values = expressionValues(":pk");+ values.set(":pk", nullString);++ assertThrows(AwsException.class, () -> DynamoDbAccessPathValidator.validateQuery(+ table, tablePath, null, "pk = :pk", null, null, null, values));++ ObjectNode multipleTypes = stringValue("p1");+ multipleTypes.put("N", "1");+ values.set(":pk", multipleTypes);++ assertThrows(AwsException.class, () -> DynamoDbAccessPathValidator.validateQuery(+ table, tablePath, null, "pk = :pk", null, null, null, values));+ }++ @Test+ void rejectsMissingKeyConditionExpressionValue() {+ ObjectNode values = expressionValues(":pk");++ AwsException error = assertThrows(AwsException.class,+ () -> DynamoDbAccessPathValidator.validateQuery(+ table, tablePath, null, "pk = :pk AND sk > :sk",+ null, null, null, values));++ assertEquals("Invalid KeyConditionExpression: An expression attribute value used in expression "+ + "is not defined; attribute value: :sk", error.getMessage());+ }++ @Test+ void rejectsMissingAttributeDefinition() {+ table.setAttributeDefinitions(null);++ AwsException error = assertThrows(AwsException.class,+ () -> validateExpression(tablePath, "pk = :pk"));++ assertEquals("ValidationException", error.getErrorCode());+ assertEquals("One or more parameter values were invalid: "+ + "Condition parameter type does not match schema type", error.getMessage());}@Test@@ -225,7 +299,25 @@ class DynamoDbAccessPathValidatorTest {private void validateExpression(DynamoDbAccessPath accessPath, String expression) {DynamoDbAccessPathValidator.validateQuery(- accessPath, null, expression, null, null, null);+ table, accessPath, null, expression, null, null, null,+ expressionValues(extractPlaceholders(expression)));+ }++ private ObjectNode expressionValues(String... placeholders) {+ ObjectNode values = mapper.createObjectNode();+ for (String placeholder : placeholders) {+ values.set(placeholder, stringValue(placeholder.substring(1)));+ }+ return values;+ }++ private String[] extractPlaceholders(String expression) {+ return java.util.regex.Pattern.compile(":\\w+")+ .matcher(expression)+ .results()+ .map(java.util.regex.MatchResult::group)+ .distinct()+ .toArray(String[]::new);}private ObjectNode legacyCondition(String operator, ArrayNode values) {@@ -238,10 +330,16 @@ class DynamoDbAccessPathValidatorTest {private ArrayNode attributeValues(String... values) {ArrayNode result = mapper.createArrayNode();for (String value : values) {- ObjectNode attributeValue = mapper.createObjectNode();- attributeValue.put("S", value);- result.add(attributeValue);+ result.add(stringValue(value));}return result;}++ private ObjectNode stringValue(String value) {+ return mapper.createObjectNode().put("S", value);+ }++ private ObjectNode numberValue(String value) {+ return mapper.createObjectNode().put("N", value);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.javaindex 579d81ba7d..1c58c2ba7c 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java@@ -253,6 +253,18 @@ class DynamoDbServiceTest {assertTrue(after.getGlobalSecondaryIndexes().isEmpty());}+ @Test+ void createTableRejectsMissingKeyAttributeDefinition() {+ AwsException error = assertThrows(AwsException.class, () -> service.createTable(+ "Users",+ List.of(new KeySchemaElement("userId", "HASH")),+ List.of(),+ 5L, 5L, "eu-west-1"));++ assertEquals("ValidationException", error.getErrorCode());+ assertEquals("Invalid KeySchema: Some index key attribute have no definition", error.getMessage());+ }+@Testvoid describeTable() {String region = "eu-west-1";