Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): bill a vector attribute at four bytes per dimension (#6688)
TATSUNO “Taz” Yasuhiro3867b5feaad7
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.javaindex 6c17a57676..d65b57e8c6 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java@@ -2,15 +2,19 @@ package io.github.hectorvent.floci.services.dynamodb;import com.fasterxml.jackson.databind.JsonNode;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;import java.math.BigDecimal;import java.nio.charset.StandardCharsets;import java.util.Base64;+import java.util.HashSet;import java.util.Iterator;import java.util.LinkedHashMap;import java.util.List;import java.util.Locale;import java.util.Map;+import java.util.Set;import java.util.function.Function;/**@@ -22,6 +26,9 @@ final class DynamoDbItemSize {static final String UPDATE_EXCEEDED = "Item size to update has exceeded the maximum allowed size";+ /** What a vector index reads per vector component, whatever the written decimal text costs. */+ static final int VECTOR_COMPONENT_BYTES = 4;+// An UpdateExpression is also measured as a statement: these per action, the target paths// with a shared prefix counted once, and every operand a SET reads or an ADD writes.// Every other surface, AttributeUpdates included, measures only the finished item.@@ -162,6 +169,33 @@ final class DynamoDbItemSize {}}+ static int billedItemSize(TableDefinition table, JsonNode item) {+ return billedItemSize(table, item, calculateItemSize(item));+ }++ /**+ * The size a read or write of the base table item is billed at. Once a vector index is+ * ACTIVE, the attribute it reads costs its name plus four bytes per dimension there, even+ * when an item written before the index holds something other than a vector. The 400KB+ * limit and every secondary index copy still count what the attribute holds.+ */+ static int billedItemSize(TableDefinition table, JsonNode item, int storedSize) {+ List<VectorIndex> indexes = table.getVectorIndexes();+ if (item == null || indexes.isEmpty()) {+ return storedSize;+ }+ int size = storedSize;+ Set<String> counted = new HashSet<>();+ for (VectorIndex index : indexes) {+ String name = index.getVectorAttributeName();+ JsonNode vector = item.get(name);+ if (vector != null && "ACTIVE".equals(index.getIndexStatus()) && counted.add(name)) {+ size += (int) (VECTOR_COMPONENT_BYTES * index.getDimensions()) - attributeValueSize(vector);+ }+ }+ return size;+ }+static int calculateItemSize(JsonNode item) {if (item == null || !item.isObject()) return 0;int total = 0;diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.javaindex 3142ef7537..de4b6a0bbe 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java@@ -122,7 +122,7 @@ class DynamoDbPartiQLHandler {if (fetchesFromTable(stmt, table, accessPath)) {for (JsonNode item : page.scannedItems()) {tableUnits += NativeDynamoDbJsonHandler.readCapacityUnits(- DynamoDbItemSize.calculateItemSize(item), consistentRead);+ DynamoDbItemSize.billedItemSize(table, item), consistentRead);}}Map<String, Double> indexUnits = Map.of(accessPath.indexName(), units);@@ -215,7 +215,7 @@ class DynamoDbPartiQLHandler {if (item == null) {return new Page(Collections.emptyList(), null, 0, List.of());}- return new Page(List.of(item), null, DynamoDbItemSize.calculateItemSize(item), List.of(item));+ return new Page(List.of(item), null, DynamoDbItemSize.billedItemSize(table, item), List.of(item));}ExprAttrBuilder eav = new ExprAttrBuilder();ExprAttrNameBuilder ean = new ExprAttrNameBuilder();diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 509eaf0d2c..025f5a64cf 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -1663,6 +1663,7 @@ public class DynamoDbService {// that you have reached the end of the result set").final int MAX_RESPONSE_BYTES = 1024 * 1024;int accSize = 0;+ long billedBytes = 0;int included = -1; // index one past the last included item; -1 = no boundary hitfor (int i = 0; i < evaluatedItems.size(); i++) {int sz = readItemSize(evaluatedItems.get(i), accessPath, table);@@ -1671,6 +1672,7 @@ public class DynamoDbService {break;}accSize += sz;+ billedBytes += billedReadSize(evaluatedItems.get(i), accessPath, table, sz);if (limit != null && limit > 0 && i + 1 >= limit) {included = i + 1; // the Limit cap stops the read AFTER this itembreak;@@ -1693,7 +1695,7 @@ public class DynamoDbService {LOG.tracev("Query on {0}: returned={1} scanned={2}",canonicalTableName, evaluatedItems.size(), scannedCount);- return new QueryResult(evaluatedItems, scannedCount, accSize, lastEvaluatedKey, scannedItems);+ return new QueryResult(evaluatedItems, scannedCount, billedBytes, lastEvaluatedKey, scannedItems);}public ScanResult scan(String tableName, String filterExpression,@@ -1748,6 +1750,7 @@ public class DynamoDbService {final int MAX_RESPONSE_BYTES = 1024 * 1024;int totalScanned = 0;int accSize = 0;+ long billedBytes = 0;List<JsonNode> results = new ArrayList<>();List<JsonNode> scannedItems = new ArrayList<>();JsonNode lastEvaluatedKey = null;@@ -1780,6 +1783,7 @@ public class DynamoDbService {break;}accSize += sz;+ billedBytes += billedReadSize(item, accessPath, table, sz);totalScanned++;scannedItems.add(item);lastScanned = item;@@ -1802,7 +1806,7 @@ public class DynamoDbService {LOG.tracev("Scan on {0}: returned={1} scanned={2}",canonicalTableName, results.size(), totalScanned);- return new ScanResult(results, totalScanned, accSize, lastEvaluatedKey, scannedItems);+ return new ScanResult(results, totalScanned, billedBytes, lastEvaluatedKey, scannedItems);}// A read served by a KEYS_ONLY or INCLUDE index is sized on the projection the@@ -1816,6 +1820,13 @@ public class DynamoDbService {(ObjectNode) item, accessPath.projectedAttributeNames(table)));}+ // The 1 MB page counts the size above. The bill of a base table read counts a vector+ // attribute at its billed size, while an index read is billed on the copy the index holds.+ private static long billedReadSize(JsonNode item, DynamoDbAccessPath accessPath, TableDefinition table,+ int readSize) {+ return accessPath.isIndex() ? readSize : DynamoDbItemSize.billedItemSize(table, item, readSize);+ }+public boolean matchesScanFilterPublic(JsonNode item, JsonNode scanFilter) {return matchesScanFilter(item, scanFilter);}@@ -2151,14 +2162,14 @@ public class DynamoDbService {}JsonNode oldItem = oldImages.get(i);JsonNode newItem = newImages.get(i);+ TableDefinition table = requireActiveTable(participant.storageKey());DynamoDbWriteCapacity.Cost cost;if (replay) {- cost = DynamoDbTransactCapacity.read(oldItem, newItem);+ cost = DynamoDbTransactCapacity.read(table, oldItem, newItem);} else if (transactItems.get(i).has("ConditionCheck")) {- cost = DynamoDbTransactCapacity.conditionCheck(oldItem);+ cost = DynamoDbTransactCapacity.conditionCheck(table, oldItem);} else {- cost = DynamoDbTransactCapacity.write(- requireActiveTable(participant.storageKey()), oldItem, newItem);+ cost = DynamoDbTransactCapacity.write(table, oldItem, newItem);}byTable.merge(participant.tableName(), cost, DynamoDbWriteCapacity.Cost::plus);}@@ -2429,7 +2440,8 @@ public class DynamoDbService {try {JsonNode item = getItem(tableName, key, region);results.add(projectTransactGet(item, get));- capacity.merge(canonicalTableName(region, tableName), DynamoDbTransactCapacity.read(item, null),+ capacity.merge(canonicalTableName(region, tableName),+ DynamoDbTransactCapacity.read(requireTableForItemCall(tableName, region), item, null),DynamoDbWriteCapacity.Cost::plus);cancelReasons.add(new TransactionCanceledException.CancellationReason("", null));} catch (AwsException e) {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.javaindex f7a6504349..99f79b8343 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java@@ -22,22 +22,22 @@ final class DynamoDbTransactCapacity {return cost.withTable(cost.table() * 2);}- static DynamoDbWriteCapacity.Cost conditionCheck(JsonNode item) {- return tableOnly(2 * units(item, WRITE_UNIT_BYTES));+ static DynamoDbWriteCapacity.Cost conditionCheck(TableDefinition table, JsonNode item) {+ return tableOnly(2 * units(table, item, WRITE_UNIT_BYTES));}- static DynamoDbWriteCapacity.Cost read(JsonNode oldItem, JsonNode newItem) {- return tableOnly(2 * Math.max(units(oldItem, READ_UNIT_BYTES), units(newItem, READ_UNIT_BYTES)));+ static DynamoDbWriteCapacity.Cost read(TableDefinition table, JsonNode oldItem, JsonNode newItem) {+ return tableOnly(2 * Math.max(units(table, oldItem, READ_UNIT_BYTES), units(table, newItem, READ_UNIT_BYTES)));}private static DynamoDbWriteCapacity.Cost tableOnly(long units) {return new DynamoDbWriteCapacity.Cost(units, Map.of(), Map.of());}- private static long units(JsonNode item, int unitBytes) {+ private static long units(TableDefinition table, JsonNode item, int unitBytes) {if (item == null) {return 1;}- return Math.max(1, (DynamoDbItemSize.calculateItemSize(item) + unitBytes - 1) / unitBytes);+ return Math.max(1, (DynamoDbItemSize.billedItemSize(table, item) + unitBytes - 1) / unitBytes);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.javaindex 9fd5504f81..4291efe365 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java@@ -29,9 +29,6 @@ final class DynamoDbWriteCapacity {/** The smallest figure VectorWriteRequestBytes reports for an entry a write changes. */private static final double VECTOR_WRITE_FLOOR_BYTES = 1024;- /** What the index stores per vector component, whatever the written decimal text costs. */- private static final int VECTOR_COMPONENT_BYTES = 4;-private DynamoDbWriteCapacity() {}record Cost(double table, Map<String, Double> gsi, Map<String, Double> lsi,@@ -72,7 +69,7 @@ final class DynamoDbWriteCapacity {/** oldItem and newItem are the stored images before and after the write; either may be null. */static Cost forWrite(TableDefinition table, JsonNode oldItem, JsonNode newItem) {- double tableUnits = (double) Math.max(1, Math.max(sizeUnits(oldItem), sizeUnits(newItem)));+ double tableUnits = (double) Math.max(1, Math.max(tableUnits(table, oldItem), tableUnits(table, newItem)));LinkedHashMap<String, Double> gsiUnits = new LinkedHashMap<>();List<GlobalSecondaryIndex> gsis = table.getGlobalSecondaryIndexes();if (gsis != null) {@@ -128,7 +125,7 @@ final class DynamoDbWriteCapacity {private static double entryBytes(VectorIndex index, JsonNode view) {String vectorAttribute = index.getVectorAttributeName();double bytes = DynamoDbItemSize.utf8Length(vectorAttribute)- + VECTOR_COMPONENT_BYTES * index.getDimensions();+ + DynamoDbItemSize.VECTOR_COMPONENT_BYTES * index.getDimensions();for (Map.Entry<String, JsonNode> field : view.properties()) {if (!vectorAttribute.equals(field.getKey())) {bytes += DynamoDbItemSize.utf8Length(field.getKey())@@ -249,10 +246,15 @@ final class DynamoDbWriteCapacity {return false;}+ private static long tableUnits(TableDefinition table, JsonNode item) {+ return item == null ? 0 : units(DynamoDbItemSize.billedItemSize(table, item));+ }+private static long sizeUnits(JsonNode image) {- if (image == null) {- return 0;- }- return Math.max(1, (DynamoDbItemSize.calculateItemSize(image) + 1023) / 1024);+ return image == null ? 0 : units(DynamoDbItemSize.calculateItemSize(image));+ }++ private static long units(int bytes) {+ return Math.max(1, (bytes + 1023) / 1024);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.javaindex 5828fa80c8..1d08525e60 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java@@ -663,7 +663,9 @@ public class NativeDynamoDbJsonHandler {}JsonNode item = dynamoDbService.getItem(tableName, key, region);- var itemBytes = item != null ? DynamoDbItemSize.calculateItemSize(item) : 0;+ int itemBytes = item != null+ ? DynamoDbItemSize.billedItemSize(dynamoDbService.requireTableForItemCall(tableName, region), item)+ : 0;if (item != null && projectionExpression != null) {item = ProjectionEvaluator.project(item, projectionExpression, exprAttrNames);} else if (item != null && attributesToGet != null) {diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorBilledSizeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorBilledSizeIntegrationTest.javanew file mode 100644index 0000000000..06f9aa1a10--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorBilledSizeIntegrationTest.java@@ -0,0 +1,176 @@+package io.github.hectorvent.floci.services.dynamodb;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.ValidatableResponse;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import java.util.Collections;++import static io.github.hectorvent.floci.testing.DynamoDbRequests.ddb;+import static org.hamcrest.Matchers.equalTo;++/**+ * A read or write of a base table item bills an attribute a vector index reads at its name+ * plus four bytes per dimension. A GSI copy of the item still counts the numbers. Every vector+ * here holds 38-digit integers, 20 bytes each as numbers, so billing by the numbers would land+ * a unit higher.+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class DynamoDbVectorBilledSizeIntegrationTest {++ private static final String TABLE = "vector-billed-size";+ private static final String WIDE = "12345678901234567890123456789012345678";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void writesBillTheVectorAtFourBytesPerDimension() {+ ddb("CreateTable", """+ {+ "TableName": "%s",+ "AttributeDefinitions": [+ {"AttributeName":"pk","AttributeType":"S"},+ {"AttributeName":"gk","AttributeType":"S"}+ ],+ "KeySchema": [{"AttributeName":"pk","KeyType":"HASH"}],+ "GlobalSecondaryIndexes": [{+ "IndexName": "gsi",+ "KeySchema": [{"AttributeName":"gk","KeyType":"HASH"}],+ "Projection": {"ProjectionType":"ALL"}+ }],+ "VectorIndexes": [+ {"IndexName":"vix","VectorAttribute":{"AttributeName":"embedding"},"Dimensions":3,+ "DistanceFunction":"COSINE","Projection":{"ProjectionType":"ALL"}},+ {"IndexName":"vkeys","VectorAttribute":{"AttributeName":"embedding"},"Dimensions":3,+ "DistanceFunction":"COSINE","Projection":{"ProjectionType":"KEYS_ONLY"}},+ {"IndexName":"vdims","VectorAttribute":{"AttributeName":"sixdimvec"},"Dimensions":6,+ "DistanceFunction":"COSINE","Projection":{"ProjectionType":"ALL"}}+ ],+ "BillingMode": "PAY_PER_REQUEST"+ }+ """.formatted(TABLE))+ .statusCode(200);++ put(item(1024, "a3", "embedding", 3, null)).body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));+ put(item(1025, "b3", "embedding", 3, null)).body("ConsumedCapacity.CapacityUnits", equalTo(2.0f));+ put(item(1024, "a6", "sixdimvec", 6, null)).body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));++ put(item(1000, "u3", "embedding", 3, null));+ ddb("UpdateItem", """+ {"TableName":"%s","Key":{"pk":{"S":"u3"}},"UpdateExpression":"SET q = :q",+ "ExpressionAttributeValues":{":q":{"S":"%s"}},"ReturnConsumedCapacity":"TOTAL"}+ """.formatted(TABLE, "x".repeat(23)))+ .statusCode(200)+ .body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));+ ddb("DeleteItem", """+ {"TableName":"%s","Key":{"pk":{"S":"a3"}},"ReturnConsumedCapacity":"TOTAL"}+ """.formatted(TABLE))+ .statusCode(200)+ .body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));+ ddb("TransactWriteItems", """+ {"TransactItems":[{"Put":{"TableName":"%s","Item":%s}}],"ReturnConsumedCapacity":"TOTAL"}+ """.formatted(TABLE, item(1024, "t3", "embedding", 3, null)))+ .statusCode(200)+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(2.0f));+ }++ @Test+ @Order(2)+ void aGsiCopyStillCountsTheNumbers() {+ ddb("PutItem", """+ {"TableName":"%s","Item":%s,"ReturnConsumedCapacity":"INDEXES"}+ """.formatted(TABLE, item(1024, "g3", "embedding", 3, "k")))+ .statusCode(200)+ .body("ConsumedCapacity.Table.CapacityUnits", equalTo(1.0f))+ .body("ConsumedCapacity.GlobalSecondaryIndexes.gsi.CapacityUnits", equalTo(2.0f));+ }++ @Test+ @Order(3)+ void readsBillTheVectorAtFourBytesPerDimension() {+ put(item(4096, "r6", "sixdimvec", 6, null));++ ddb("GetItem", """+ {"TableName":"%s","Key":{"pk":{"S":"r6"}},"ConsistentRead":true,"ReturnConsumedCapacity":"TOTAL"}+ """.formatted(TABLE))+ .statusCode(200)+ .body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));+ ddb("Query", """+ {"TableName":"%s","KeyConditionExpression":"pk = :p","ExpressionAttributeValues":{":p":{"S":"r6"}},+ "ConsistentRead":true,"ReturnConsumedCapacity":"TOTAL"}+ """.formatted(TABLE))+ .statusCode(200)+ .body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));+ ddb("TransactGetItems", """+ {"TransactItems":[{"Get":{"TableName":"%s","Key":{"pk":{"S":"r6"}}}}],"ReturnConsumedCapacity":"TOTAL"}+ """.formatted(TABLE))+ .statusCode(200)+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(2.0f));+ }++ @Test+ @Order(4)+ void anIndexAddedLaterBillsWhateverTheAttributeHoldsOnceActive() {+ String table = TABLE + "-later";+ ddb("CreateTable", """+ {"TableName":"%s","AttributeDefinitions":[{"AttributeName":"pk","AttributeType":"S"}],+ "KeySchema":[{"AttributeName":"pk","KeyType":"HASH"}],"BillingMode":"PAY_PER_REQUEST"}+ """.formatted(table))+ .statusCode(200);+ // 4084 bytes besides the string, so 4096 once it costs 12 bytes as a 3 dimension vector.+ ddb("PutItem", """+ {"TableName":"%s","Item":{"pk":{"S":"s"},"embedding":{"S":"%s"},"pad":{"S":"%s"}}}+ """.formatted(table, "x".repeat(2000), "x".repeat(4084 - 2 - 1 - 9 - 3)))+ .statusCode(200);+ String get = """+ {"TableName":"%s","Key":{"pk":{"S":"s"}},"ConsistentRead":true,"ReturnConsumedCapacity":"TOTAL"}+ """.formatted(table);++ ddb("UpdateTable", """+ {"TableName":"%s","VectorIndexUpdates":[{"Create":{"IndexName":"vix",+ "VectorAttribute":{"AttributeName":"embedding"},"Dimensions":3,"DistanceFunction":"COSINE",+ "Projection":{"ProjectionType":"KEYS_ONLY"}}}]}+ """.formatted(table))+ .statusCode(200);+ ddb("GetItem", get).statusCode(200).body("ConsumedCapacity.CapacityUnits", equalTo(2.0f));++ ddb("DescribeTable", """+ {"TableName":"%s"}+ """.formatted(table))+ .statusCode(200)+ .body("Table.VectorIndexes[0].IndexStatus", equalTo("ACTIVE"));+ ddb("GetItem", get).statusCode(200).body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));+ }++ private static ValidatableResponse put(String item) {+ return ddb("PutItem", """+ {"TableName":"%s","Item":%s,"ReturnConsumedCapacity":"TOTAL"}+ """.formatted(TABLE, item))+ .statusCode(200);+ }++ /** An item measuring {@code billedBytes} with its vector at four bytes per dimension. */+ private static String item(int billedBytes, String key, String attribute, int dims, String gk) {+ String vector = String.join(",", Collections.nCopies(dims, "{\"N\":\"" + WIDE + "\"}"));+ String gsiKey = gk == null ? "" : ",\"gk\":{\"S\":\"" + gk + "\"}";+ return """+ {"pk":{"S":"%s"},"%s":{"L":[%s]}%s,"pad":{"S":"%s"}}"""+ .formatted(key, attribute, vector, gsiKey, padFor(billedBytes, key, attribute, dims, gk));+ }++ private static String padFor(int billedBytes, String key, String attribute, int dims, String gk) {+ int fixed = "pk".length() + key.length() + attribute.length() + 4 * dims + "pad".length()+ + (gk == null ? 0 : "gk".length() + gk.length());+ return "x".repeat(billedBytes - fixed);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.javaindex 7b242f03a6..a997362417 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java@@ -260,6 +260,42 @@ class DynamoDbWriteCapacityTest {assertTrue(cost.vectorBytes().isEmpty());}+ @Test+ void tableUnitsCountTheVectorAtFourBytesPerDimensionOnce() {+ ObjectNode wide = item("""+ {"pk": {"S": "a3"},+ "embedding": {"L": [{"N": "%1$s"}, {"N": "%1$s"}, {"N": "%1$s"}]}}+ """.formatted("12345678901234567890123456789012345678"));+ // 4 for the key, 9 for the vector's name and 12 for its dimensions leave 999 for the pad.+ wide.set("pad", item("""+ {"S": "%s"}+ """.formatted("x".repeat(996))));+ assertTrue(DynamoDbItemSize.calculateItemSize(wide) > 1024, "the stored numbers take it past 1KB");++ assertEquals(1024, DynamoDbItemSize.billedItemSize(vectorTable(), wide));+ assertEquals(1.0, DynamoDbWriteCapacity.forWrite(vectorTable(), null, wide).table());+ }++ @Test+ void anyValueUnderTheVectorAttributeCostsFourBytesPerDimension() {+ ObjectNode written = item("""+ {"pk": {"S": "s1"}, "embedding": {"S": "%s"}}+ """.formatted("x".repeat(2000)));+ // 4 for the key, 9 for the vector's name and 12 for its dimensions.+ assertEquals(25, DynamoDbItemSize.billedItemSize(vectorTable(), written));+ }++ @Test+ void aVectorIndexStillCreatingLeavesTheStoredSize() {+ TableDefinition creating = vectorTable();+ creating.getVectorIndexes().forEach(index -> index.setIndexStatus("CREATING"));+ ObjectNode written = item("""+ {"pk": {"S": "s1"}, "embedding": {"S": "%s"}}+ """.formatted("x".repeat(2000)));+ assertEquals(DynamoDbItemSize.calculateItemSize(written),+ DynamoDbItemSize.billedItemSize(creating, written));+ }+@Testvoid itemMissingTheSearchSchemaHashEntersNoVectorIndex() {TableDefinition partitioned = new TableDefinition();