Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): bill a vector attribute at four bytes per dimension (#6688)

TATSUNO “Taz” Yasuhiro3867b5feaad7
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java
index 6c17a57676..d65b57e8c6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java
@@ -2,15 +2,19 @@ package io.github.hectorvent.floci.services.dynamodb;
import com.fasterxml.jackson.databind.JsonNode;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.function.Function;
/**
@@ -22,6 +26,9 @@ final class DynamoDbItemSize {
static final String UPDATE_EXCEEDED = "Item size to update has exceeded the maximum allowed size";
+ /** What a vector index reads per vector component, whatever the written decimal text costs. */
+ static final int VECTOR_COMPONENT_BYTES = 4;
+
// An UpdateExpression is also measured as a statement: these per action, the target paths
// with a shared prefix counted once, and every operand a SET reads or an ADD writes.
// Every other surface, AttributeUpdates included, measures only the finished item.
@@ -162,6 +169,33 @@ final class DynamoDbItemSize {
}
}
+ static int billedItemSize(TableDefinition table, JsonNode item) {
+ return billedItemSize(table, item, calculateItemSize(item));
+ }
+
+ /**
+ * The size a read or write of the base table item is billed at. Once a vector index is
+ * ACTIVE, the attribute it reads costs its name plus four bytes per dimension there, even
+ * when an item written before the index holds something other than a vector. The 400KB
+ * limit and every secondary index copy still count what the attribute holds.
+ */
+ static int billedItemSize(TableDefinition table, JsonNode item, int storedSize) {
+ List<VectorIndex> indexes = table.getVectorIndexes();
+ if (item == null || indexes.isEmpty()) {
+ return storedSize;
+ }
+ int size = storedSize;
+ Set<String> counted = new HashSet<>();
+ for (VectorIndex index : indexes) {
+ String name = index.getVectorAttributeName();
+ JsonNode vector = item.get(name);
+ if (vector != null && "ACTIVE".equals(index.getIndexStatus()) && counted.add(name)) {
+ size += (int) (VECTOR_COMPONENT_BYTES * index.getDimensions()) - attributeValueSize(vector);
+ }
+ }
+ return size;
+ }
+
static int calculateItemSize(JsonNode item) {
if (item == null || !item.isObject()) return 0;
int total = 0;
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java
index 3142ef7537..de4b6a0bbe 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java
@@ -122,7 +122,7 @@ class DynamoDbPartiQLHandler {
if (fetchesFromTable(stmt, table, accessPath)) {
for (JsonNode item : page.scannedItems()) {
tableUnits += NativeDynamoDbJsonHandler.readCapacityUnits(
- DynamoDbItemSize.calculateItemSize(item), consistentRead);
+ DynamoDbItemSize.billedItemSize(table, item), consistentRead);
}
}
Map<String, Double> indexUnits = Map.of(accessPath.indexName(), units);
@@ -215,7 +215,7 @@ class DynamoDbPartiQLHandler {
if (item == null) {
return new Page(Collections.emptyList(), null, 0, List.of());
}
- return new Page(List.of(item), null, DynamoDbItemSize.calculateItemSize(item), List.of(item));
+ return new Page(List.of(item), null, DynamoDbItemSize.billedItemSize(table, item), List.of(item));
}
ExprAttrBuilder eav = new ExprAttrBuilder();
ExprAttrNameBuilder ean = new ExprAttrNameBuilder();
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 509eaf0d2c..025f5a64cf 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -1663,6 +1663,7 @@ public class DynamoDbService {
// that you have reached the end of the result set").
final int MAX_RESPONSE_BYTES = 1024 * 1024;
int accSize = 0;
+ long billedBytes = 0;
int included = -1; // index one past the last included item; -1 = no boundary hit
for (int i = 0; i < evaluatedItems.size(); i++) {
int sz = readItemSize(evaluatedItems.get(i), accessPath, table);
@@ -1671,6 +1672,7 @@ public class DynamoDbService {
break;
}
accSize += sz;
+ billedBytes += billedReadSize(evaluatedItems.get(i), accessPath, table, sz);
if (limit != null && limit > 0 && i + 1 >= limit) {
included = i + 1; // the Limit cap stops the read AFTER this item
break;
@@ -1693,7 +1695,7 @@ public class DynamoDbService {
LOG.tracev("Query on {0}: returned={1} scanned={2}",
canonicalTableName, evaluatedItems.size(), scannedCount);
- return new QueryResult(evaluatedItems, scannedCount, accSize, lastEvaluatedKey, scannedItems);
+ return new QueryResult(evaluatedItems, scannedCount, billedBytes, lastEvaluatedKey, scannedItems);
}
public ScanResult scan(String tableName, String filterExpression,
@@ -1748,6 +1750,7 @@ public class DynamoDbService {
final int MAX_RESPONSE_BYTES = 1024 * 1024;
int totalScanned = 0;
int accSize = 0;
+ long billedBytes = 0;
List<JsonNode> results = new ArrayList<>();
List<JsonNode> scannedItems = new ArrayList<>();
JsonNode lastEvaluatedKey = null;
@@ -1780,6 +1783,7 @@ public class DynamoDbService {
break;
}
accSize += sz;
+ billedBytes += billedReadSize(item, accessPath, table, sz);
totalScanned++;
scannedItems.add(item);
lastScanned = item;
@@ -1802,7 +1806,7 @@ public class DynamoDbService {
LOG.tracev("Scan on {0}: returned={1} scanned={2}",
canonicalTableName, results.size(), totalScanned);
- return new ScanResult(results, totalScanned, accSize, lastEvaluatedKey, scannedItems);
+ return new ScanResult(results, totalScanned, billedBytes, lastEvaluatedKey, scannedItems);
}
// A read served by a KEYS_ONLY or INCLUDE index is sized on the projection the
@@ -1816,6 +1820,13 @@ public class DynamoDbService {
(ObjectNode) item, accessPath.projectedAttributeNames(table)));
}
+ // The 1 MB page counts the size above. The bill of a base table read counts a vector
+ // attribute at its billed size, while an index read is billed on the copy the index holds.
+ private static long billedReadSize(JsonNode item, DynamoDbAccessPath accessPath, TableDefinition table,
+ int readSize) {
+ return accessPath.isIndex() ? readSize : DynamoDbItemSize.billedItemSize(table, item, readSize);
+ }
+
public boolean matchesScanFilterPublic(JsonNode item, JsonNode scanFilter) {
return matchesScanFilter(item, scanFilter);
}
@@ -2151,14 +2162,14 @@ public class DynamoDbService {
}
JsonNode oldItem = oldImages.get(i);
JsonNode newItem = newImages.get(i);
+ TableDefinition table = requireActiveTable(participant.storageKey());
DynamoDbWriteCapacity.Cost cost;
if (replay) {
- cost = DynamoDbTransactCapacity.read(oldItem, newItem);
+ cost = DynamoDbTransactCapacity.read(table, oldItem, newItem);
} else if (transactItems.get(i).has("ConditionCheck")) {
- cost = DynamoDbTransactCapacity.conditionCheck(oldItem);
+ cost = DynamoDbTransactCapacity.conditionCheck(table, oldItem);
} else {
- cost = DynamoDbTransactCapacity.write(
- requireActiveTable(participant.storageKey()), oldItem, newItem);
+ cost = DynamoDbTransactCapacity.write(table, oldItem, newItem);
}
byTable.merge(participant.tableName(), cost, DynamoDbWriteCapacity.Cost::plus);
}
@@ -2429,7 +2440,8 @@ public class DynamoDbService {
try {
JsonNode item = getItem(tableName, key, region);
results.add(projectTransactGet(item, get));
- capacity.merge(canonicalTableName(region, tableName), DynamoDbTransactCapacity.read(item, null),
+ capacity.merge(canonicalTableName(region, tableName),
+ DynamoDbTransactCapacity.read(requireTableForItemCall(tableName, region), item, null),
DynamoDbWriteCapacity.Cost::plus);
cancelReasons.add(new TransactionCanceledException.CancellationReason("", null));
} catch (AwsException e) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java
index f7a6504349..99f79b8343 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java
@@ -22,22 +22,22 @@ final class DynamoDbTransactCapacity {
return cost.withTable(cost.table() * 2);
}
- static DynamoDbWriteCapacity.Cost conditionCheck(JsonNode item) {
- return tableOnly(2 * units(item, WRITE_UNIT_BYTES));
+ static DynamoDbWriteCapacity.Cost conditionCheck(TableDefinition table, JsonNode item) {
+ return tableOnly(2 * units(table, item, WRITE_UNIT_BYTES));
}
- static DynamoDbWriteCapacity.Cost read(JsonNode oldItem, JsonNode newItem) {
- return tableOnly(2 * Math.max(units(oldItem, READ_UNIT_BYTES), units(newItem, READ_UNIT_BYTES)));
+ static DynamoDbWriteCapacity.Cost read(TableDefinition table, JsonNode oldItem, JsonNode newItem) {
+ return tableOnly(2 * Math.max(units(table, oldItem, READ_UNIT_BYTES), units(table, newItem, READ_UNIT_BYTES)));
}
private static DynamoDbWriteCapacity.Cost tableOnly(long units) {
return new DynamoDbWriteCapacity.Cost(units, Map.of(), Map.of());
}
- private static long units(JsonNode item, int unitBytes) {
+ private static long units(TableDefinition table, JsonNode item, int unitBytes) {
if (item == null) {
return 1;
}
- return Math.max(1, (DynamoDbItemSize.calculateItemSize(item) + unitBytes - 1) / unitBytes);
+ return Math.max(1, (DynamoDbItemSize.billedItemSize(table, item) + unitBytes - 1) / unitBytes);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java
index 9fd5504f81..4291efe365 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java
@@ -29,9 +29,6 @@ final class DynamoDbWriteCapacity {
/** The smallest figure VectorWriteRequestBytes reports for an entry a write changes. */
private static final double VECTOR_WRITE_FLOOR_BYTES = 1024;
- /** What the index stores per vector component, whatever the written decimal text costs. */
- private static final int VECTOR_COMPONENT_BYTES = 4;
-
private DynamoDbWriteCapacity() {}
record Cost(double table, Map<String, Double> gsi, Map<String, Double> lsi,
@@ -72,7 +69,7 @@ final class DynamoDbWriteCapacity {
/** oldItem and newItem are the stored images before and after the write; either may be null. */
static Cost forWrite(TableDefinition table, JsonNode oldItem, JsonNode newItem) {
- double tableUnits = (double) Math.max(1, Math.max(sizeUnits(oldItem), sizeUnits(newItem)));
+ double tableUnits = (double) Math.max(1, Math.max(tableUnits(table, oldItem), tableUnits(table, newItem)));
LinkedHashMap<String, Double> gsiUnits = new LinkedHashMap<>();
List<GlobalSecondaryIndex> gsis = table.getGlobalSecondaryIndexes();
if (gsis != null) {
@@ -128,7 +125,7 @@ final class DynamoDbWriteCapacity {
private static double entryBytes(VectorIndex index, JsonNode view) {
String vectorAttribute = index.getVectorAttributeName();
double bytes = DynamoDbItemSize.utf8Length(vectorAttribute)
- + VECTOR_COMPONENT_BYTES * index.getDimensions();
+ + DynamoDbItemSize.VECTOR_COMPONENT_BYTES * index.getDimensions();
for (Map.Entry<String, JsonNode> field : view.properties()) {
if (!vectorAttribute.equals(field.getKey())) {
bytes += DynamoDbItemSize.utf8Length(field.getKey())
@@ -249,10 +246,15 @@ final class DynamoDbWriteCapacity {
return false;
}
+ private static long tableUnits(TableDefinition table, JsonNode item) {
+ return item == null ? 0 : units(DynamoDbItemSize.billedItemSize(table, item));
+ }
+
private static long sizeUnits(JsonNode image) {
- if (image == null) {
- return 0;
- }
- return Math.max(1, (DynamoDbItemSize.calculateItemSize(image) + 1023) / 1024);
+ return image == null ? 0 : units(DynamoDbItemSize.calculateItemSize(image));
+ }
+
+ private static long units(int bytes) {
+ return Math.max(1, (bytes + 1023) / 1024);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
index 5828fa80c8..1d08525e60 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
@@ -663,7 +663,9 @@ public class NativeDynamoDbJsonHandler {
}
JsonNode item = dynamoDbService.getItem(tableName, key, region);
- var itemBytes = item != null ? DynamoDbItemSize.calculateItemSize(item) : 0;
+ int itemBytes = item != null
+ ? DynamoDbItemSize.billedItemSize(dynamoDbService.requireTableForItemCall(tableName, region), item)
+ : 0;
if (item != null && projectionExpression != null) {
item = ProjectionEvaluator.project(item, projectionExpression, exprAttrNames);
} else if (item != null && attributesToGet != null) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorBilledSizeIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorBilledSizeIntegrationTest.java
new file mode 100644
index 0000000000..06f9aa1a10
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorBilledSizeIntegrationTest.java
@@ -0,0 +1,176 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.ValidatableResponse;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.util.Collections;
+
+import static io.github.hectorvent.floci.testing.DynamoDbRequests.ddb;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * A read or write of a base table item bills an attribute a vector index reads at its name
+ * plus four bytes per dimension. A GSI copy of the item still counts the numbers. Every vector
+ * here holds 38-digit integers, 20 bytes each as numbers, so billing by the numbers would land
+ * a unit higher.
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class DynamoDbVectorBilledSizeIntegrationTest {
+
+ private static final String TABLE = "vector-billed-size";
+ private static final String WIDE = "12345678901234567890123456789012345678";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void writesBillTheVectorAtFourBytesPerDimension() {
+ ddb("CreateTable", """
+ {
+ "TableName": "%s",
+ "AttributeDefinitions": [
+ {"AttributeName":"pk","AttributeType":"S"},
+ {"AttributeName":"gk","AttributeType":"S"}
+ ],
+ "KeySchema": [{"AttributeName":"pk","KeyType":"HASH"}],
+ "GlobalSecondaryIndexes": [{
+ "IndexName": "gsi",
+ "KeySchema": [{"AttributeName":"gk","KeyType":"HASH"}],
+ "Projection": {"ProjectionType":"ALL"}
+ }],
+ "VectorIndexes": [
+ {"IndexName":"vix","VectorAttribute":{"AttributeName":"embedding"},"Dimensions":3,
+ "DistanceFunction":"COSINE","Projection":{"ProjectionType":"ALL"}},
+ {"IndexName":"vkeys","VectorAttribute":{"AttributeName":"embedding"},"Dimensions":3,
+ "DistanceFunction":"COSINE","Projection":{"ProjectionType":"KEYS_ONLY"}},
+ {"IndexName":"vdims","VectorAttribute":{"AttributeName":"sixdimvec"},"Dimensions":6,
+ "DistanceFunction":"COSINE","Projection":{"ProjectionType":"ALL"}}
+ ],
+ "BillingMode": "PAY_PER_REQUEST"
+ }
+ """.formatted(TABLE))
+ .statusCode(200);
+
+ put(item(1024, "a3", "embedding", 3, null)).body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));
+ put(item(1025, "b3", "embedding", 3, null)).body("ConsumedCapacity.CapacityUnits", equalTo(2.0f));
+ put(item(1024, "a6", "sixdimvec", 6, null)).body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));
+
+ put(item(1000, "u3", "embedding", 3, null));
+ ddb("UpdateItem", """
+ {"TableName":"%s","Key":{"pk":{"S":"u3"}},"UpdateExpression":"SET q = :q",
+ "ExpressionAttributeValues":{":q":{"S":"%s"}},"ReturnConsumedCapacity":"TOTAL"}
+ """.formatted(TABLE, "x".repeat(23)))
+ .statusCode(200)
+ .body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));
+ ddb("DeleteItem", """
+ {"TableName":"%s","Key":{"pk":{"S":"a3"}},"ReturnConsumedCapacity":"TOTAL"}
+ """.formatted(TABLE))
+ .statusCode(200)
+ .body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));
+ ddb("TransactWriteItems", """
+ {"TransactItems":[{"Put":{"TableName":"%s","Item":%s}}],"ReturnConsumedCapacity":"TOTAL"}
+ """.formatted(TABLE, item(1024, "t3", "embedding", 3, null)))
+ .statusCode(200)
+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(2.0f));
+ }
+
+ @Test
+ @Order(2)
+ void aGsiCopyStillCountsTheNumbers() {
+ ddb("PutItem", """
+ {"TableName":"%s","Item":%s,"ReturnConsumedCapacity":"INDEXES"}
+ """.formatted(TABLE, item(1024, "g3", "embedding", 3, "k")))
+ .statusCode(200)
+ .body("ConsumedCapacity.Table.CapacityUnits", equalTo(1.0f))
+ .body("ConsumedCapacity.GlobalSecondaryIndexes.gsi.CapacityUnits", equalTo(2.0f));
+ }
+
+ @Test
+ @Order(3)
+ void readsBillTheVectorAtFourBytesPerDimension() {
+ put(item(4096, "r6", "sixdimvec", 6, null));
+
+ ddb("GetItem", """
+ {"TableName":"%s","Key":{"pk":{"S":"r6"}},"ConsistentRead":true,"ReturnConsumedCapacity":"TOTAL"}
+ """.formatted(TABLE))
+ .statusCode(200)
+ .body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));
+ ddb("Query", """
+ {"TableName":"%s","KeyConditionExpression":"pk = :p","ExpressionAttributeValues":{":p":{"S":"r6"}},
+ "ConsistentRead":true,"ReturnConsumedCapacity":"TOTAL"}
+ """.formatted(TABLE))
+ .statusCode(200)
+ .body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));
+ ddb("TransactGetItems", """
+ {"TransactItems":[{"Get":{"TableName":"%s","Key":{"pk":{"S":"r6"}}}}],"ReturnConsumedCapacity":"TOTAL"}
+ """.formatted(TABLE))
+ .statusCode(200)
+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(2.0f));
+ }
+
+ @Test
+ @Order(4)
+ void anIndexAddedLaterBillsWhateverTheAttributeHoldsOnceActive() {
+ String table = TABLE + "-later";
+ ddb("CreateTable", """
+ {"TableName":"%s","AttributeDefinitions":[{"AttributeName":"pk","AttributeType":"S"}],
+ "KeySchema":[{"AttributeName":"pk","KeyType":"HASH"}],"BillingMode":"PAY_PER_REQUEST"}
+ """.formatted(table))
+ .statusCode(200);
+ // 4084 bytes besides the string, so 4096 once it costs 12 bytes as a 3 dimension vector.
+ ddb("PutItem", """
+ {"TableName":"%s","Item":{"pk":{"S":"s"},"embedding":{"S":"%s"},"pad":{"S":"%s"}}}
+ """.formatted(table, "x".repeat(2000), "x".repeat(4084 - 2 - 1 - 9 - 3)))
+ .statusCode(200);
+ String get = """
+ {"TableName":"%s","Key":{"pk":{"S":"s"}},"ConsistentRead":true,"ReturnConsumedCapacity":"TOTAL"}
+ """.formatted(table);
+
+ ddb("UpdateTable", """
+ {"TableName":"%s","VectorIndexUpdates":[{"Create":{"IndexName":"vix",
+ "VectorAttribute":{"AttributeName":"embedding"},"Dimensions":3,"DistanceFunction":"COSINE",
+ "Projection":{"ProjectionType":"KEYS_ONLY"}}}]}
+ """.formatted(table))
+ .statusCode(200);
+ ddb("GetItem", get).statusCode(200).body("ConsumedCapacity.CapacityUnits", equalTo(2.0f));
+
+ ddb("DescribeTable", """
+ {"TableName":"%s"}
+ """.formatted(table))
+ .statusCode(200)
+ .body("Table.VectorIndexes[0].IndexStatus", equalTo("ACTIVE"));
+ ddb("GetItem", get).statusCode(200).body("ConsumedCapacity.CapacityUnits", equalTo(1.0f));
+ }
+
+ private static ValidatableResponse put(String item) {
+ return ddb("PutItem", """
+ {"TableName":"%s","Item":%s,"ReturnConsumedCapacity":"TOTAL"}
+ """.formatted(TABLE, item))
+ .statusCode(200);
+ }
+
+ /** An item measuring {@code billedBytes} with its vector at four bytes per dimension. */
+ private static String item(int billedBytes, String key, String attribute, int dims, String gk) {
+ String vector = String.join(",", Collections.nCopies(dims, "{\"N\":\"" + WIDE + "\"}"));
+ String gsiKey = gk == null ? "" : ",\"gk\":{\"S\":\"" + gk + "\"}";
+ return """
+ {"pk":{"S":"%s"},"%s":{"L":[%s]}%s,"pad":{"S":"%s"}}"""
+ .formatted(key, attribute, vector, gsiKey, padFor(billedBytes, key, attribute, dims, gk));
+ }
+
+ private static String padFor(int billedBytes, String key, String attribute, int dims, String gk) {
+ int fixed = "pk".length() + key.length() + attribute.length() + 4 * dims + "pad".length()
+ + (gk == null ? 0 : "gk".length() + gk.length());
+ return "x".repeat(billedBytes - fixed);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java
index 7b242f03a6..a997362417 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java
@@ -260,6 +260,42 @@ class DynamoDbWriteCapacityTest {
assertTrue(cost.vectorBytes().isEmpty());
}
+ @Test
+ void tableUnitsCountTheVectorAtFourBytesPerDimensionOnce() {
+ ObjectNode wide = item("""
+ {"pk": {"S": "a3"},
+ "embedding": {"L": [{"N": "%1$s"}, {"N": "%1$s"}, {"N": "%1$s"}]}}
+ """.formatted("12345678901234567890123456789012345678"));
+ // 4 for the key, 9 for the vector's name and 12 for its dimensions leave 999 for the pad.
+ wide.set("pad", item("""
+ {"S": "%s"}
+ """.formatted("x".repeat(996))));
+ assertTrue(DynamoDbItemSize.calculateItemSize(wide) > 1024, "the stored numbers take it past 1KB");
+
+ assertEquals(1024, DynamoDbItemSize.billedItemSize(vectorTable(), wide));
+ assertEquals(1.0, DynamoDbWriteCapacity.forWrite(vectorTable(), null, wide).table());
+ }
+
+ @Test
+ void anyValueUnderTheVectorAttributeCostsFourBytesPerDimension() {
+ ObjectNode written = item("""
+ {"pk": {"S": "s1"}, "embedding": {"S": "%s"}}
+ """.formatted("x".repeat(2000)));
+ // 4 for the key, 9 for the vector's name and 12 for its dimensions.
+ assertEquals(25, DynamoDbItemSize.billedItemSize(vectorTable(), written));
+ }
+
+ @Test
+ void aVectorIndexStillCreatingLeavesTheStoredSize() {
+ TableDefinition creating = vectorTable();
+ creating.getVectorIndexes().forEach(index -> index.setIndexStatus("CREATING"));
+ ObjectNode written = item("""
+ {"pk": {"S": "s1"}, "embedding": {"S": "%s"}}
+ """.formatted("x".repeat(2000)));
+ assertEquals(DynamoDbItemSize.calculateItemSize(written),
+ DynamoDbItemSize.billedItemSize(creating, written));
+ }
+
@Test
void itemMissingTheSearchSchemaHashEntersNoVectorIndex() {
TableDefinition partitioned = new TableDefinition();