Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): reject UpdateTable GSI changes AWS refuses (#4714)
Sam Fakhreddinefdb21de0953f
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex e60a87950e..c95b93aca0 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -2316,7 +2316,8 @@ public class DynamoDbService {if (newAttrDefs != null) {knownAttrDefs.addAll(newAttrDefs);}- validateVectorIndexUpdates(table, vectorIndexCreates, vectorIndexDeletes, knownAttrDefs,+ validateVectorIndexUpdates(table, vectorIndexCreates, vectorIndexDeletes,+ gsiCreates.size() + gsiDeletes.size(), knownAttrDefs,billingMode != null ? billingMode : table.getBillingMode());if (readCapacity != null) {@@ -2372,13 +2373,14 @@ public class DynamoDbService {* current lifecycle state.** <p>AWS allows one online index operation per table at a time. That limit binds the table,- * not the request, so two creates in one request and a create issued while an earlier index+ * not the request, and GSI creates and deletes count against it too, so two creates in one+ * request and a create issued while an earlier index* still builds are refused the same way. Deleting an index that is still in its resource* allocation phase is refused with a different error, and the same delete is accepted once* the index reaches backfilling.*/private void validateVectorIndexUpdates(TableDefinition table, List<VectorIndexCreate> creates,- List<String> deletes,+ List<String> deletes, int gsiOperations,List<AttributeDefinition> attributeDefinitions,String billingMode) {validateVectorIndexMembers(creates);@@ -2389,7 +2391,7 @@ public class DynamoDbService {"One or more parameter values were invalid: Vector indexes are only supported "+ "for PAY_PER_REQUEST tables", 400);}- if (creates.isEmpty() && deletes.isEmpty()) {+ if (creates.isEmpty() && deletes.isEmpty() && gsiOperations == 0) {return;}for (VectorIndexCreate create : creates) {@@ -2421,7 +2423,8 @@ public class DynamoDbService {.count();// Deleting an index that is still backfilling takes over the slot that index already// holds, so it does not need one of its own.- long onlineOperations = building + creates.size() + deletes.size() - deletesOfBuildingIndexes;+ long onlineOperations = building + creates.size() + deletes.size() + gsiOperations+ - deletesOfBuildingIndexes;if (onlineOperations > 1) {throw new AwsException("LimitExceededException","Subscriber limit exceeded: Only 1 online index can be created or deleted "diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.javaindex 7698f027bd..b69dc16c5b 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java@@ -1755,6 +1755,12 @@ public class NativeDynamoDbJsonHandler {}}+ if ((gsiCreates.size() + gsiDeletes.size() > 0) && (request.has("BillingMode")+ || request.has("ProvisionedThroughput") || request.has("OnDemandThroughput"))) {+ throw new AwsException("ValidationException",+ "You cannot create or delete index while updating table IOPS", 400);+ }+if (!gsiThroughputUpdates.isEmpty()) {TableDefinition currentTable = dynamoDbService.describeTable(tableName, region);for (GsiThroughputUpdate gsiUpdate : gsiThroughputUpdates) {diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.javaindex 12d0c844d7..a93a624a14 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java@@ -1709,6 +1709,7 @@ class DynamoDbIntegrationTest {@Test@Order(25)void updateTableDeleteAllGsis() {+ // AWS deletes at most one GSI per UpdateTable request (#4572), so the two go one at a time.given().header("X-Amz-Target", "DynamoDB_20120810.UpdateTable").contentType(DYNAMODB_CONTENT_TYPE)@@ -1720,7 +1721,23 @@ class DynamoDbIntegrationTest {"Delete": {"IndexName": "OwnerIndex"}- },+ }+ ]+ }+ """)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("TableDescription.GlobalSecondaryIndexes.size()", equalTo(1));++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "GsiTestTable",+ "GlobalSecondaryIndexUpdates": [{"Delete": {"IndexName": "OwnerIndexProj"@@ -5796,4 +5813,272 @@ given().when().post("/").then().statusCode(200);}++ @Test+ void gsiDeletionTogetherWithABillingModeChangeFailsWithValidation() {+ // Catches: Floci deletes the index and switches the billing mode in one request instead of rejecting it+ String tableName = "GsiDeleteWithBillingTable";++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],+ "AttributeDefinitions": [+ {"AttributeName": "pk", "AttributeType": "S"},+ {"AttributeName": "gsiPk", "AttributeType": "S"}+ ],+ "BillingMode": "PAY_PER_REQUEST",+ "GlobalSecondaryIndexes": [+ {+ "IndexName": "GsiOne",+ "KeySchema": [{"AttributeName": "gsiPk", "KeyType": "HASH"}],+ "Projection": {"ProjectionType": "ALL"}+ }+ ]+ }+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "BillingMode": "PROVISIONED",+ "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},+ "GlobalSecondaryIndexUpdates": [{"Delete": {"IndexName": "GsiOne"}}]+ }+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("You cannot create or delete index while updating table IOPS"));++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {"TableName": "%s"}+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Table.GlobalSecondaryIndexes.size()", equalTo(1))+ .body("Table.BillingModeSummary.BillingMode", equalTo("PAY_PER_REQUEST"));++ deleteTable(tableName);+ }++ @Test+ void twoGsiDeletionsInSameRequestFailWithLimitExceeded() {+ // Catches: Floci deletes both indexes when one UpdateTable request carries two GSI deletes+ String tableName = "TwoGsiDeleteTable";++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],+ "AttributeDefinitions": [+ {"AttributeName": "pk", "AttributeType": "S"},+ {"AttributeName": "gsiPk", "AttributeType": "S"},+ {"AttributeName": "gsiSk", "AttributeType": "S"}+ ],+ "BillingMode": "PAY_PER_REQUEST",+ "GlobalSecondaryIndexes": [+ {+ "IndexName": "GsiOne",+ "KeySchema": [{"AttributeName": "gsiPk", "KeyType": "HASH"}, {"AttributeName": "gsiSk", "KeyType": "RANGE"}],+ "Projection": {"ProjectionType": "ALL"}+ },+ {+ "IndexName": "GsiTwo",+ "KeySchema": [{"AttributeName": "gsiPk", "KeyType": "HASH"}, {"AttributeName": "gsiSk", "KeyType": "RANGE"}],+ "Projection": {"ProjectionType": "ALL"}+ }+ ]+ }+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "GlobalSecondaryIndexUpdates": [+ {"Delete": {"IndexName": "GsiOne"}},+ {"Delete": {"IndexName": "GsiTwo"}}+ ]+ }+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("LimitExceededException"))+ .body("message", equalTo("Subscriber limit exceeded: Only 1 online index can be created "+ + "or deleted simultaneously per table"));++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {"TableName": "%s"}+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Table.GlobalSecondaryIndexes.size()", equalTo(2));++ deleteTable(tableName);+ }++ @Test+ void twoGsiCreationsInSameRequestFailWithLimitExceeded() {+ // Catches: Floci builds both indexes when one UpdateTable request carries two GSI creates+ String tableName = "TwoGsiCreateTable";+ createOnDemandTableForGsiUpdates(tableName);++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "AttributeDefinitions": [{"AttributeName": "gsiPk", "AttributeType": "S"}],+ "GlobalSecondaryIndexUpdates": [+ {"Create": {"IndexName": "GsiOne", "KeySchema": [{"AttributeName": "gsiPk", "KeyType": "HASH"}], "Projection": {"ProjectionType": "ALL"}}},+ {"Create": {"IndexName": "GsiTwo", "KeySchema": [{"AttributeName": "gsiPk", "KeyType": "HASH"}], "Projection": {"ProjectionType": "ALL"}}}+ ]+ }+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("LimitExceededException"))+ .body("message", equalTo("Subscriber limit exceeded: Only 1 online index can be created "+ + "or deleted simultaneously per table"));++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {"TableName": "%s"}+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Table.GlobalSecondaryIndexes", nullValue());++ deleteTable(tableName);+ }++ @Test+ void gsiCreationWithTableOnDemandThroughputFailsAndLeavesLimitsUnchanged() {+ // Catches: an index create applied together with new table-level OnDemandThroughput limits+ String tableName = "GsiCreateOnDemandTable";+ createOnDemandTableForGsiUpdates(tableName);++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "AttributeDefinitions": [{"AttributeName": "gsiPk", "AttributeType": "S"}],+ "OnDemandThroughput": {"MaxReadRequestUnits": 100, "MaxWriteRequestUnits": 100},+ "GlobalSecondaryIndexUpdates": [+ {"Create": {"IndexName": "GsiOne", "KeySchema": [{"AttributeName": "gsiPk", "KeyType": "HASH"}], "Projection": {"ProjectionType": "ALL"}}}+ ]+ }+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("You cannot create or delete index while updating table IOPS"));++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {"TableName": "%s"}+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("Table.GlobalSecondaryIndexes", nullValue())+ .body("Table.OnDemandThroughput", nullValue());++ deleteTable(tableName);+ }++ @Test+ void gsiCreationWithTableProvisionedThroughputAloneFails() {+ // Catches: the IOPS conflict only fires when BillingMode is sent together with throughput+ String tableName = "GsiCreateProvisionedTable";+ createOnDemandTableForGsiUpdates(tableName);++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "AttributeDefinitions": [{"AttributeName": "gsiPk", "AttributeType": "S"}],+ "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},+ "GlobalSecondaryIndexUpdates": [+ {"Create": {"IndexName": "GsiOne", "KeySchema": [{"AttributeName": "gsiPk", "KeyType": "HASH"}], "Projection": {"ProjectionType": "ALL"}}}+ ]+ }+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ValidationException"))+ .body("message", equalTo("You cannot create or delete index while updating table IOPS"));++ deleteTable(tableName);+ }++ private void createOnDemandTableForGsiUpdates(String tableName) {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],+ "BillingMode": "PAY_PER_REQUEST"+ }+ """.formatted(tableName))+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorIndexLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorIndexLifecycleTest.javaindex e8b7fc5a27..1882ff4fa3 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorIndexLifecycleTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorIndexLifecycleTest.java@@ -4,6 +4,7 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;@@ -166,6 +167,26 @@ class DynamoDbVectorIndexLifecycleTest {assertTrue(service.describeTable(TABLE, REGION).getVectorIndexes().isEmpty());}+ @Test+ void gsiCreateAlongsideAVectorIndexCreateExceedsTheOnlineIndexLimit() {+ // Catches: the online index limit counts GSI and vector index operations separately,+ // so one of each in a single request gets through+ GlobalSecondaryIndex gsi = new GlobalSecondaryIndex("byTitle",+ List.of(new KeySchemaElement("title", "HASH")), null, "ALL", List.of());++ AwsException refused = assertThrows(AwsException.class, () ->+ service.updateTable(TABLE, null, null, List.of(gsi), List.of(),+ List.of(new AttributeDefinition("title", "S")),+ List.of(create("one", "vectorIndexUpdates.1.member.create")),+ List.of(), null, REGION));++ assertEquals("LimitExceededException", refused.getErrorCode());+ assertEquals("Subscriber limit exceeded: Only 1 online index can be created or deleted "+ + "simultaneously per table", refused.getMessage());+ assertTrue(service.describeTable(TABLE, REGION).getVectorIndexes().isEmpty());+ assertTrue(service.describeTable(TABLE, REGION).getGlobalSecondaryIndexes().isEmpty());+ }+@Testvoid secondCreateWhileAnIndexBuildsExceedsTheOnlineIndexLimit() {addIndex("first");