Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(dynamodb): vector indexes and SearchVectors (#4215)

TATSUNO “Taz” Yasuhiro133ef62d2725
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbTest.java
index 3bc6ea5c6d..22f5dfffa2 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbTest.java
@@ -7,6 +7,7 @@ import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.BatchGetItemRequest;
import software.amazon.awssdk.services.dynamodb.model.BatchGetItemResponse;
import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemRequest;
+import software.amazon.awssdk.services.dynamodb.model.BillingMode;
import software.amazon.awssdk.services.dynamodb.model.CreateReplicationGroupMemberAction;
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
import software.amazon.awssdk.services.dynamodb.model.CreateTableResponse;
@@ -20,12 +21,15 @@ import software.amazon.awssdk.services.dynamodb.model.DescribeTimeToLiveRequest;
import software.amazon.awssdk.services.dynamodb.model.DescribeTimeToLiveResponse;
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
+import software.amazon.awssdk.services.dynamodb.model.IndexStatus;
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
import software.amazon.awssdk.services.dynamodb.model.KeysAndAttributes;
import software.amazon.awssdk.services.dynamodb.model.KeyType;
import software.amazon.awssdk.services.dynamodb.model.ListTablesResponse;
import software.amazon.awssdk.services.dynamodb.model.ListTagsOfResourceRequest;
import software.amazon.awssdk.services.dynamodb.model.ListTagsOfResourceResponse;
+import software.amazon.awssdk.services.dynamodb.model.Projection;
+import software.amazon.awssdk.services.dynamodb.model.ProjectionType;
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.PutRequest;
@@ -36,6 +40,9 @@ import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
+import software.amazon.awssdk.services.dynamodb.model.SearchResultItem;
+import software.amazon.awssdk.services.dynamodb.model.SearchVectorsRequest;
+import software.amazon.awssdk.services.dynamodb.model.SearchVectorsResponse;
import software.amazon.awssdk.services.dynamodb.model.TableStatus;
import software.amazon.awssdk.services.dynamodb.model.TagResourceRequest;
import software.amazon.awssdk.services.dynamodb.model.TimeToLiveStatus;
@@ -45,6 +52,10 @@ import software.amazon.awssdk.services.dynamodb.model.UpdateItemResponse;
import software.amazon.awssdk.services.dynamodb.model.UpdateReplicationGroupMemberAction;
import software.amazon.awssdk.services.dynamodb.model.UpdateTableRequest;
import software.amazon.awssdk.services.dynamodb.model.UpdateTableResponse;
+import software.amazon.awssdk.services.dynamodb.model.VectorAttributeDefinition;
+import software.amazon.awssdk.services.dynamodb.model.VectorDistanceFunction;
+import software.amazon.awssdk.services.dynamodb.model.VectorIndex;
+import software.amazon.awssdk.services.dynamodb.model.VectorIndexDescription;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
import java.util.List;
@@ -58,6 +69,8 @@ class DynamoDbTest {
private static DynamoDbClient ddb;
private static final String TABLE_NAME = "sdk-test-table";
+ private static final String VECTOR_TABLE_NAME = "sdk-test-vector-table";
+ private static final String VECTOR_INDEX_NAME = "embedding-index";
private static String tableArn;
@BeforeAll
@@ -71,6 +84,10 @@ class DynamoDbTest {
try {
ddb.deleteTable(DeleteTableRequest.builder().tableName(TABLE_NAME).build());
} catch (Exception ignored) {}
+ // Nothing to delete when searchVectors did not get as far as creating it.
+ try {
+ ddb.deleteTable(DeleteTableRequest.builder().tableName(VECTOR_TABLE_NAME).build());
+ } catch (Exception ignored) {}
ddb.close();
}
}
@@ -388,4 +405,86 @@ class DynamoDbTest {
ListTablesResponse response = ddb.listTables();
assertThat(response.tableNames()).doesNotContain(TABLE_NAME);
}
+
+ @Test
+ @Order(20)
+ void searchVectors() {
+ ddb.createTable(CreateTableRequest.builder()
+ .tableName(VECTOR_TABLE_NAME)
+ .keySchema(KeySchemaElement.builder()
+ .attributeName("docId").keyType(KeyType.HASH).build())
+ .attributeDefinitions(AttributeDefinition.builder()
+ .attributeName("docId").attributeType(ScalarAttributeType.S).build())
+ .billingMode(BillingMode.PAY_PER_REQUEST)
+ .vectorIndexes(VectorIndex.builder()
+ .indexName(VECTOR_INDEX_NAME)
+ .vectorAttribute(VectorAttributeDefinition.builder()
+ .attributeName("embedding").build())
+ .projection(Projection.builder().projectionType(ProjectionType.ALL).build())
+ .dimensions(3L)
+ .distanceFunction(VectorDistanceFunction.COSINE)
+ .build())
+ .build());
+
+ DescribeTableResponse described = ddb.describeTable(
+ DescribeTableRequest.builder().tableName(VECTOR_TABLE_NAME).build());
+ VectorIndexDescription index = described.table().vectorIndexes().get(0);
+ assertThat(index.indexName()).isEqualTo(VECTOR_INDEX_NAME);
+ assertThat(index.dimensions()).isEqualTo(3L);
+ assertThat(index.distanceFunction()).isEqualTo(VectorDistanceFunction.COSINE);
+ assertThat(index.indexStatus()).isEqualTo(IndexStatus.ACTIVE);
+ assertThat(index.backfilling()).isNull();
+
+ putVector("near", "1", "0", "0");
+ putVector("mid", "0.6", "0.8", "0");
+ putVector("far", "-1", "0", "0");
+
+ SearchVectorsResponse ranked = ddb.searchVectors(SearchVectorsRequest.builder()
+ .tableName(VECTOR_TABLE_NAME)
+ .indexName(VECTOR_INDEX_NAME)
+ .searchVector(number("1"), number("0"), number("0"))
+ .topK(3)
+ .build());
+
+ assertThat(ranked.searchResults())
+ .extracting(result -> result.item().get("docId").s())
+ .containsExactly("near", "mid", "far");
+ assertThat(ranked.searchResults())
+ .extracting(SearchResultItem::score)
+ .containsExactly(0.0, 0.3999999761581421, 2.0);
+ assertThat(ranked.searchResults().get(0).item())
+ .containsKey("title")
+ .doesNotContainKey("embedding");
+
+ SearchVectorsResponse projected = ddb.searchVectors(SearchVectorsRequest.builder()
+ .tableName(VECTOR_TABLE_NAME)
+ .indexName(VECTOR_INDEX_NAME)
+ .searchVector(number("1"), number("0"), number("0"))
+ .topK(1)
+ .projectionExpression("docId, embedding")
+ .build());
+
+ Map<String, AttributeValue> item = projected.searchResults().get(0).item();
+ assertThat(item).containsOnlyKeys("docId", "embedding");
+ // The index serves its own 32 bit copy, so "1" comes back as "1.0".
+ assertThat(item.get("embedding").l())
+ .extracting(AttributeValue::n)
+ .containsExactly("1.0", "0.0", "0.0");
+ }
+
+ private static void putVector(String docId, String x, String y, String z) {
+ ddb.putItem(PutItemRequest.builder()
+ .tableName(VECTOR_TABLE_NAME)
+ .item(Map.of(
+ "docId", AttributeValue.builder().s(docId).build(),
+ "title", AttributeValue.builder().s("title " + docId).build(),
+ "embedding", AttributeValue.builder()
+ .l(number(x), number(y), number(z)).build()
+ ))
+ .build());
+ }
+
+ private static AttributeValue number(String value) {
+ return AttributeValue.builder().n(value).build();
+ }
}
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index 5fd64dfab2..c229505181 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -18,6 +18,7 @@
| `UpdateItem` | Partially update an item |
| `Query` | Query by partition key with optional filter |
| `Scan` | Full table scan with optional filter |
+| `SearchVectors` | Rank a vector index by distance from a query vector |
| `BatchWriteItem` | Write/delete up to 25 items across tables |
| `BatchGetItem` | Read up to 100 items across tables |
| `TransactWriteItems` | ACID write transaction |
@@ -61,6 +62,8 @@ landing table, checkpoint, and retry behavior.
| `FLOCI_SERVICES_DYNAMODB_ENABLED` | `true` | Enable or disable the service |
| `FLOCI_STORAGE_SERVICES_DYNAMODB_MODE` | *(global default)* | Storage mode override for DynamoDB (`memory`, `persistent`, `hybrid`, `wal`) |
| `FLOCI_STORAGE_SERVICES_DYNAMODB_FLUSH_INTERVAL_MS` | `5000` | Flush interval for `hybrid`/`wal` storage modes (milliseconds) |
+| `FLOCI_SERVICES_DYNAMODB_VECTOR_INDEX_ALLOCATION_SECONDS` | `4` | Seconds a vector index added by `UpdateTable` spends in resource allocation |
+| `FLOCI_SERVICES_DYNAMODB_VECTOR_INDEX_BACKFILL_SECONDS` | `10` | Seconds that index then spends backfilling before it goes `ACTIVE` |
### Storage and Performance
@@ -142,6 +145,79 @@ aws dynamodb create-table \
--endpoint-url $AWS_ENDPOINT_URL
```
+Deviations from AWS:
+
+- **No index is stored.** A `Query` or `Scan` on an index reads the base table and applies the
+ index's key schema and projection on the way out.
+- **Index reads are immediately consistent.** A GSI on AWS is eventually consistent, so code that
+ tolerates replication lag never exercises that wait here.
+- **A new index is `ACTIVE` at once.** AWS copies the table's items into a new index first, which
+ takes minutes on a large table. Nothing is copied here, so a waiter returns straight away.
+
+## Vector indexes
+
+A vector index serves `SearchVectors`, which ranks a table's items by the distance between a query
+vector and the vector attribute each item carries. Indexes are declared on `CreateTable` or added
+later with `UpdateTable`, on `PAY_PER_REQUEST` tables only. An item that does not carry the vector
+attribute, or that lacks the index's `HASH` search schema attribute, is still written but stays out
+of that index. `Query`, `Scan` and PartiQL cannot read a vector index.
+
+```bash
+# A table with a vector index
+aws dynamodb create-table \
+ --table-name Docs \
+ --attribute-definitions AttributeName=docId,AttributeType=S \
+ --key-schema AttributeName=docId,KeyType=HASH \
+ --billing-mode PAY_PER_REQUEST \
+ --vector-indexes '[{
+ "IndexName": "embedding-index",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"
+ }]' \
+ --endpoint-url $AWS_ENDPOINT_URL
+
+# Write an item carrying a vector
+aws dynamodb put-item \
+ --table-name Docs \
+ --item '{"docId":{"S":"d1"},"title":{"S":"first"},"embedding":{"L":[{"N":"1"},{"N":"0"},{"N":"0"}]}}' \
+ --endpoint-url $AWS_ENDPOINT_URL
+
+# Search it
+aws dynamodb search-vectors \
+ --table-name Docs \
+ --index-name embedding-index \
+ --search-vector '[{"N":"1"},{"N":"0"},{"N":"0"}]' \
+ --top-k 5 \
+ --endpoint-url $AWS_ENDPOINT_URL
+```
+
+`DistanceFunction` is `COSINE`, `EUCLIDEAN` or `DOT_PRODUCT`. The first two rank the lowest score
+first, `DOT_PRODUCT` the highest. The vector attribute is left out of a result unless
+`--projection-expression` names it, and when it is named the values returned are the index's own
+32 bit copies, so a `1` written to the table comes back as `1.0`.
+
+Deviations from AWS:
+
+- **The search is exact, not approximate.** AWS may return a slightly different set or order on a
+ large index. Scoring every item makes a result here repeatable.
+- **No index is stored, as above.** Every search reads the vectors from the base table and converts
+ them, so the work grows with items times dimensions. A large index is slower than AWS.
+- **A written vector is searchable at once.** AWS copies it into the index in the background, so
+ code that polls for a new vector never waits here.
+- **`ItemCount` and `IndexSizeBytes` always report 0.** That is what AWS reports for a fresh index,
+ because it refreshes both roughly every six hours.
+- **An index created with its table is `ACTIVE` at once.** One added by `UpdateTable` walks a
+ resource allocation phase and then a backfill phase. Their lengths are
+ `FLOCI_SERVICES_DYNAMODB_VECTOR_INDEX_ALLOCATION_SECONDS` and
+ `FLOCI_SERVICES_DYNAMODB_VECTOR_INDEX_BACKFILL_SECONDS`, 4 and 10 seconds by default, against
+ minutes on AWS.
+- **`SearchVectors` is served on Floci's ordinary endpoint.** AWS gives the operation a dedicated
+ search endpoint. The SDKs honor an endpoint override, so this is invisible to callers.
+- **`VectorSearchRequestBytes` is a fixed figure, not a measured one.** It uses the same 1024 byte
+ floor AWS uses. AWS's own value above that floor is not deterministic.
+
## Export to S3
Export table data to an S3 bucket as gzip-compressed NDJSON (DynamoDB JSON format):
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index b4a6547ede..00dd134b6e 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1283,6 +1283,24 @@ public interface EmulatorConfig {
interface DynamoDbServiceConfig {
@WithDefault("true")
boolean enabled();
+
+ /**
+ * Seconds a vector index added by UpdateTable spends in the resource allocation phase,
+ * where the table reads UPDATING, the index reads CREATING with Backfilling false, and a
+ * delete of that index is refused.
+ * Env: FLOCI_SERVICES_DYNAMODB_VECTOR_INDEX_ALLOCATION_SECONDS
+ */
+ @WithDefault("4")
+ int vectorIndexAllocationSeconds();
+
+ /**
+ * Seconds the same index then spends backfilling, where the table reads ACTIVE and the
+ * index reads CREATING with Backfilling true. A vector index created by CreateTable skips
+ * both phases and is ACTIVE at once, which is what AWS reports there.
+ * Env: FLOCI_SERVICES_DYNAMODB_VECTOR_INDEX_BACKFILL_SECONDS
+ */
+ @WithDefault("10")
+ int vectorIndexBackfillSeconds();
}
interface SnsServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java
index 1be75fa0ae..6373e4f358 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java
@@ -153,7 +153,7 @@ final class DynamoDbItemSize {
return 0;
}
- private static int utf8Length(String s) {
+ static int utf8Length(String s) {
if (s == null) return 0;
return s.getBytes(StandardCharsets.UTF_8).length;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
index 0e5ee42f5d..6f2441b9b2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
@@ -57,6 +57,7 @@ public class DynamoDbJsonHandler {
case "UpdateItem" -> handleUpdateItem(request, region);
case "Query" -> handleQuery(request, region);
case "Scan" -> handleScan(request, region);
+ case "SearchVectors" -> handleSearchVectors(request, region);
case "BatchWriteItem" -> handleBatchWriteItem(request, region);
case "BatchGetItem" -> handleBatchGetItem(request, region);
case "UpdateTable" -> handleUpdateTable(request, region);
@@ -191,6 +192,18 @@ public class DynamoDbJsonHandler {
}
}
+ List<DynamoDbService.VectorIndexCreate> vectorIndexes = new ArrayList<>();
+ JsonNode vectorIndexArray = request.path("VectorIndexes");
+ if (vectorIndexArray.isArray()) {
+ int vectorPosition = 0;
+ for (JsonNode vectorIndexNode : vectorIndexArray) {
+ vectorPosition++;
+ String memberPath = "vectorIndexes." + vectorPosition + ".member";
+ vectorIndexes.add(new DynamoDbService.VectorIndexCreate(
+ parseVectorIndex(vectorIndexNode, memberPath), memberPath));
+ }
+ }
+
String billingMode = request.has("BillingMode")
? request.get("BillingMode").asText() : null;
@@ -221,7 +234,7 @@ public class DynamoDbJsonHandler {
}
TableDefinition table = dynamoDbService.createTable(tableName, keySchema, attrDefs,
- readCapacity, writeCapacity, gsis, lsis, region);
+ readCapacity, writeCapacity, gsis, lsis, vectorIndexes, billingMode, region);
table.setTableStatus(initialStatus);
table.setDeletionProtectionEnabled(deletionProtection);
@@ -305,6 +318,48 @@ public class DynamoDbJsonHandler {
}
}
+ /** Reads a VectorIndex or the CreateVectorIndexAction of a VectorIndexUpdate; same members. */
+ private static VectorIndex parseVectorIndex(JsonNode node, String memberPath) {
+ // An absent VectorAttribute is reported as the member itself, one that is present but
+ // carries no AttributeName as the nested member. Only the request shape tells them apart.
+ JsonNode vectorAttribute = node.path("VectorAttribute");
+ if (vectorAttribute.isObject() && !vectorAttribute.hasNonNull("AttributeName")) {
+ throw new AwsException("ValidationException",
+ "1 validation error detected: Value null at '" + memberPath
+ + ".vectorAttribute.attributeName' failed to satisfy constraint: "
+ + "Member must not be null", 400);
+ }
+ List<SearchSchemaElement> searchSchema = new ArrayList<>();
+ JsonNode searchSchemaArray = node.path("SearchSchema");
+ if (searchSchemaArray.isArray()) {
+ for (JsonNode element : searchSchemaArray) {
+ searchSchema.add(new SearchSchemaElement(
+ element.path("AttributeName").asText(null),
+ element.path("SearchSchemaElementType").asText(null)));
+ }
+ }
+ String projectionType = node.hasNonNull("Projection")
+ ? node.path("Projection").path("ProjectionType").asText("ALL") : null;
+ List<String> nonKeyAttributes = new ArrayList<>();
+ JsonNode nonKeyAttrArray = node.path("Projection").path("NonKeyAttributes");
+ if (nonKeyAttrArray.isArray()) {
+ for (JsonNode nonKeyAttr : nonKeyAttrArray) {
+ nonKeyAttributes.add(nonKeyAttr.asText());
+ }
+ }
+ rejectEmptyNonKeyAttributes(nonKeyAttrArray, memberPath);
+ validateProjectionSpec(projectionType, nonKeyAttributes);
+ Long dimensions = node.hasNonNull("Dimensions") ? node.get("Dimensions").asLong() : null;
+ return new VectorIndex(
+ node.path("IndexName").asText(null),
+ node.path("VectorAttribute").path("AttributeName").asText(null),
+ searchSchema,
+ projectionType,
+ nonKeyAttributes,
+ dimensions,
+ node.path("DistanceFunction").asText(null));
+ }
+
// INCLUDE requires the NonKeyAttributes list; every other projection type forbids it.
private static void validateProjectionSpec(String projectionType, List<String> nonKeyAttributes) {
if ("INCLUDE".equals(projectionType) && nonKeyAttributes.isEmpty()) {
@@ -1180,6 +1235,111 @@ public class DynamoDbJsonHandler {
return Response.ok(response).build();
}
+ // The floor AWS reports for a small search. The real figure is non-deterministic there,
+ // so every search reports the floor (measured in eu-west-2, 2026-09-23).
+ private static final double VECTOR_SEARCH_REQUEST_BYTES = 1024;
+
+ private static final int MAX_VECTOR_SEARCH_TOP_K = 100;
+
+ private Response handleSearchVectors(JsonNode request, String region) {
+ requireSearchVectorsMember(request, "SearchVector");
+ requireSearchVectorsMember(request, "IndexName");
+ String tableName = request.path("TableName").asText();
+ String indexName = request.get("IndexName").asText();
+ String rccSearch = request.has("ReturnConsumedCapacity")
+ ? request.get("ReturnConsumedCapacity").asText() : null;
+ if (rccSearch != null && !VALID_RETURN_CONSUMED_CAPACITY.contains(rccSearch)) {
+ // No "N validation errors detected" envelope here, unlike every other operation.
+ throw new AwsException("ValidationException",
+ "Value '" + rccSearch + "' at 'returnConsumedCapacity' "
+ + "failed to satisfy constraint: Member must satisfy enum value set: "
+ + "[INDEXES, TOTAL, NONE]", 400);
+ }
+ requireSearchVectorsMember(request, "TopK");
+ int topK = request.get("TopK").asInt();
+ if (topK < 1) {
+ // AWS names this member TopK and quotes no value, unlike every other member of this
+ // family. Measured on real DynamoDB, eu-west-2, 2026-09-23.
+ throw new AwsException("ValidationException",
+ "1 validation error detected: Value at 'TopK' failed to satisfy "
+ + "constraint: Member must have value greater than or equal to 1", 400);
+ }
+ // Both bounds answer before the table and the index are resolved: AWS reports the range
+ // even for a table that does not exist.
+ if (topK > MAX_VECTOR_SEARCH_TOP_K) {
+ throw new AwsException("ValidationException",
+ "Provided TopK value '" + topK + "' is out of valid range. "
+ + "The value must be between 1 and " + MAX_VECTOR_SEARCH_TOP_K + " inclusive", 400);
+ }
+
+ TableDefinition table = dynamoDbService.settledTable(tableName, region);
+ VectorIndex index = requireSearchableVectorIndex(table, indexName);
+
+ JsonNode exprAttrNames = request.has("ExpressionAttributeNames")
+ ? request.get("ExpressionAttributeNames") : null;
+ JsonNode exprAttrValues = request.has("ExpressionAttributeValues")
+ ? request.get("ExpressionAttributeValues") : null;
+ List<DynamoDbVectorSearch.Hit> hits = DynamoDbVectorSearch.search(table, index,
+ dynamoDbService.liveItems(table, region),
+ request.path("SearchVector"), topK,
+ request.has("SearchConditionExpression")
+ ? request.get("SearchConditionExpression").asText() : null,
+ exprAttrNames, exprAttrValues,
+ request.has("ProjectionExpression") ? request.get("ProjectionExpression").asText() : null);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ ArrayNode searchResults = objectMapper.createArrayNode();
+ for (DynamoDbVectorSearch.Hit hit : hits) {
+ ObjectNode searchResult = objectMapper.createObjectNode();
+ searchResult.set("Item", hit.item());
+ searchResult.put("Score", hit.score());
+ searchResults.add(searchResult);
+ }
+ response.set("SearchResults", searchResults);
+ if (rccSearch != null && !"NONE".equals(rccSearch)) {
+ ObjectNode consumedCapacity = objectMapper.createObjectNode();
+ consumedCapacity.put("VectorSearchRequestBytes", VECTOR_SEARCH_REQUEST_BYTES);
+ response.set("ConsumedCapacity", consumedCapacity);
+ }
+ return Response.ok(response).build();
+ }
+
+ /**
+ * Rejects an omitted required member of SearchVectors the way the AWS frontend does: as a
+ * deserialisation failure quoting the offset of the body's closing brace, which for the
+ * compact JSON an SDK sends is the body length. Measured in eu-west-2, 2026-09-23.
+ */
+ private static void requireSearchVectorsMember(JsonNode request, String member) {
+ if (!request.hasNonNull(member)) {
+ throw new AwsException("ValidationException",
+ "missing field `" + member + "` at line 1 column "
+ + request.toString().length(), 400);
+ }
+ }
+
+ /**
+ * The vector index a search may run against.
+ *
+ * <p>An index still in its resource allocation phase answers as if the table did not have it,
+ * which is the wording AWS uses there and what the documented readiness wait retries on.
+ */
+ private VectorIndex requireSearchableVectorIndex(TableDefinition table, String indexName) {
+ VectorIndex index = table.findVectorIndex(indexName).orElse(null);
+ if (index == null) {
+ throw new AwsException("ValidationException",
+ "The table does not have the specified index: " + indexName, 400);
+ }
+ if ("CREATING".equals(index.getIndexStatus())) {
+ if (!dynamoDbService.isVectorIndexBackfilling(index)) {
+ throw new AwsException("ValidationException",
+ "The table does not have the specified index: " + indexName, 400);
+ }
+ throw new AwsException("ValidationException",
+ "Cannot search backfilling vector index: " + indexName, 400);
+ }
+ return index;
+ }
+
private Response handleBatchWriteItem(JsonNode request, String region) {
JsonNode requestItems = request.get("RequestItems");
if (requestItems == null || requestItems.isNull() || requestItems.isMissingNode()
@@ -1467,6 +1627,28 @@ public class DynamoDbJsonHandler {
}
}
+ List<DynamoDbService.VectorIndexCreate> vectorCreates = new ArrayList<>();
+ List<String> vectorDeletes = new ArrayList<>();
+ JsonNode vectorIndexUpdates = request.path("VectorIndexUpdates");
+ if (vectorIndexUpdates.isArray()) {
+ // AWS reports a per-member constraint under the position in this array, which a
+ // create shares with the deletes beside it.
+ int updatePosition = 0;
+ for (JsonNode update : vectorIndexUpdates) {
+ updatePosition++;
+ JsonNode createNode = update.path("Create");
+ if (createNode.isObject()) {
+ String memberPath = "vectorIndexUpdates." + updatePosition + ".member.create";
+ vectorCreates.add(new DynamoDbService.VectorIndexCreate(
+ parseVectorIndex(createNode, memberPath), memberPath));
+ }
+ JsonNode deleteNode = update.path("Delete");
+ if (deleteNode.isObject()) {
+ vectorDeletes.add(deleteNode.path("IndexName").asText(null));
+ }
+ }
+ }
+
List<AttributeDefinition> newAttrDefs = new ArrayList<>();
JsonNode attrDefsNode = request.path("AttributeDefinitions");
if (!attrDefsNode.isMissingNode() && attrDefsNode.isArray()) {
@@ -1495,7 +1677,8 @@ public class DynamoDbJsonHandler {
}
TableDefinition table = dynamoDbService.updateTable(tableName, readCapacity, writeCapacity,
- gsiCreates, gsiDeletes, newAttrDefs, region);
+ gsiCreates, gsiDeletes, newAttrDefs, vectorCreates, vectorDeletes,
+ billingModeCheck, region);
for (JsonNode updateNode : gsiUpdatesToApply) {
GlobalSecondaryIndex gsi = table.findGsi(updateNode.path("IndexName").asText()).orElseThrow();
@@ -2338,10 +2521,24 @@ public class DynamoDbJsonHandler {
if (!cost.lsi().isEmpty()) {
cc.set("LocalSecondaryIndexes", capacityUnitsMap(cost.lsi(), unitsField));
}
+ if (!cost.vectorBytes().isEmpty()) {
+ cc.set("VectorIndexes", vectorWriteBytesMap(cost.vectorBytes()));
+ }
}
return cc;
}
+ /** A vector index meters bytes processed, so it carries no CapacityUnits of its own. */
+ private ObjectNode vectorWriteBytesMap(Map<String, Double> bytesByIndex) {
+ ObjectNode node = objectMapper.createObjectNode();
+ bytesByIndex.forEach((indexName, bytes) -> {
+ ObjectNode entry = objectMapper.createObjectNode();
+ entry.put("VectorWriteRequestBytes", bytes);
+ node.set(indexName, entry);
+ });
+ return node;
+ }
+
private ObjectNode capacityUnitsMap(Map<String, Double> unitsByIndex, String unitsField) {
ObjectNode node = objectMapper.createObjectNode();
unitsByIndex.forEach((indexName, units) -> node.set(indexName, capacityUnitsNode(units, unitsField)));
@@ -2597,6 +2794,15 @@ public class DynamoDbJsonHandler {
node.set("LocalSecondaryIndexes", lsiArray);
}
+ List<VectorIndex> vectorIndexes = table.getVectorIndexes();
+ if (!vectorIndexes.isEmpty()) {
+ ArrayNode vectorIndexArray = objectMapper.createArrayNode();
+ for (VectorIndex vectorIndex : vectorIndexes) {
+ vectorIndexArray.add(vectorIndexToNode(vectorIndex));
+ }
+ node.set("VectorIndexes", vectorIndexArray);
+ }
+
if (table.getStreamArn() != null) {
ObjectNode streamSpecNode = objectMapper.createObjectNode();
streamSpecNode.put("StreamEnabled", table.isStreamEnabled());
@@ -2612,6 +2818,54 @@ public class DynamoDbJsonHandler {
return node;
}
+ /**
+ * Renders a VectorIndexDescription. ItemCount and IndexSizeBytes are always 0. AWS refreshes
+ * those two roughly every six hours, so a freshly built index reports 0 for both there too.
+ */
+ private ObjectNode vectorIndexToNode(VectorIndex vectorIndex) {
+ ObjectNode vectorIndexNode = objectMapper.createObjectNode();
+ vectorIndexNode.put("IndexName", vectorIndex.getIndexName());
+
+ ObjectNode vectorAttribute = objectMapper.createObjectNode();
+ vectorAttribute.put("AttributeName", vectorIndex.getVectorAttributeName());
+ vectorIndexNode.set("VectorAttribute", vectorAttribute);
+
+ List<SearchSchemaElement> searchSchema = vectorIndex.getSearchSchema();
+ if (searchSchema != null && !searchSchema.isEmpty()) {
+ ArrayNode searchSchemaArray = objectMapper.createArrayNode();
+ for (SearchSchemaElement element : searchSchema) {
+ ObjectNode elementNode = objectMapper.createObjectNode();
+ elementNode.put("AttributeName", element.getAttributeName());
+ elementNode.put("SearchSchemaElementType", element.getSearchSchemaElementType());
+ searchSchemaArray.add(elementNode);
+ }
+ vectorIndexNode.set("SearchSchema", searchSchemaArray);
+ }
+
+ ObjectNode projection = objectMapper.createObjectNode();
+ projection.put("ProjectionType",
+ vectorIndex.getProjectionType() != null ? vectorIndex.getProjectionType() : "ALL");
+ if ("INCLUDE".equals(vectorIndex.getProjectionType())) {
+ ArrayNode nonKeyAttributes = objectMapper.createArrayNode();
+ for (String attr : vectorIndex.getNonKeyAttributes()) {
+ nonKeyAttributes.add(attr);
+ }
+ projection.set("NonKeyAttributes", nonKeyAttributes);
+ }
+ vectorIndexNode.set("Projection", projection);
+
+ vectorIndexNode.put("Dimensions", vectorIndex.getDimensions());
+ vectorIndexNode.put("DistanceFunction", vectorIndex.getDistanceFunction());
+ vectorIndexNode.put("IndexStatus", vectorIndex.getIndexStatus());
+ if (dynamoDbService.reportsVectorIndexBackfilling(vectorIndex)) {
+ vectorIndexNode.put("Backfilling", dynamoDbService.isVectorIndexBackfilling(vectorIndex));
+ }
+ vectorIndexNode.put("IndexSizeBytes", 0);
+ vectorIndexNode.put("ItemCount", 0);
+ vectorIndexNode.put("IndexArn", vectorIndex.getIndexArn());
+ return vectorIndexNode;
+ }
+
private String defaultKmsMasterKeyArn(String region) {
return AwsArnUtils.Arn.of("kms", region, DEFAULT_ACCOUNT_ID,
"key/aws-managed-dynamodb").toString();
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java
index b632edbcbb..2fb950bdc0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLHandler.java
@@ -46,6 +46,12 @@ class DynamoDbPartiQLHandler {
private Result executeSelect(Stmt.Select stmt, PartiQLExecuteContext ctx, String region) {
TableDefinition table = service.describeTable(stmt.table(), region);
+ // A vector index is reachable through SearchVectors alone, and PartiQL says so
+ // rather than reporting the index as missing.
+ if (table.findVectorIndex(stmt.index()).isPresent()) {
+ throw new AwsException("ValidationException",
+ "Scan operation not supported on this index type", 400);
+ }
// ExecuteStatement omits the index name from the missing-index message,
// unlike Query/Scan (characterised on real AWS, eu-west-1, 2026-09-02).
DynamoDbAccessPath accessPath = DynamoDbAccessPath.resolve(table, stmt.index(),
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 4806df5a86..3251544225 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.dynamodb;
+import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
@@ -17,6 +18,7 @@ import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;
import io.github.hectorvent.floci.services.dynamodb.model.ConditionalCheckFailedException;
import io.github.hectorvent.floci.services.s3.S3Service;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -114,6 +116,26 @@ public class DynamoDbService implements ResourceProvider {
private static final int MAX_MULTI_ATTRIBUTE_KEY_PART_SIZE = 4;
+ private static final int MAX_VECTOR_INDEXES_PER_TABLE = 5;
+ private static final int MAX_VECTOR_DIMENSIONS = 4096;
+ private static final int MIN_VECTOR_INDEX_NAME_LENGTH = 3;
+ // The order AWS prints the enum constraint in, which is neither alphabetical nor the order
+ // the API reference lists.
+ private static final List<String> VECTOR_DISTANCE_FUNCTIONS = List.of(
+ DynamoDbVectorScoring.DOT_PRODUCT,
+ DynamoDbVectorScoring.COSINE,
+ DynamoDbVectorScoring.EUCLIDEAN);
+
+ /**
+ * A vector index a request adds, with the request-model path AWS reports its per-member
+ * constraints under.
+ *
+ * <p>CreateTable and UpdateTable name different members, and an UpdateTable create shares the
+ * {@code VectorIndexUpdates} array with the deletes beside it, so only the caller that read
+ * the request can build the path.
+ */
+ public record VectorIndexCreate(VectorIndex index, String memberPath) {}
+
private record IdempotencyEntry(String requestHash, long insertedAtNanos,
CompletableFuture<Map<String, DynamoDbWriteCapacity.Cost>> replayCapacity) {}
private final RegionResolver regionResolver;
@@ -121,6 +143,8 @@ public class DynamoDbService implements ResourceProvider {
private DynamoDbStreamService streamService;
private KinesisStreamingForwarder kinesisForwarder;
private S3Service s3Service;
+ private final int vectorIndexAllocationSeconds;
+ private final int vectorIndexBackfillSeconds;
private static final long MAX_DYNAMODB_LIST_INDEX = 4_294_967_294L;
@Inject
@@ -128,7 +152,8 @@ public class DynamoDbService implements ResourceProvider {
DynamoDbStreamService streamService,
KinesisStreamingForwarder kinesisForwarder,
S3Service s3Service,
- ObjectMapper objectMapper) {
+ ObjectMapper objectMapper,
+ EmulatorConfig config) {
this(storageFactory.create("dynamodb", "dynamodb-tables.json",
new TypeReference<Map<String, TableDefinition>>() {}),
storageFactory.create("dynamodb", "dynamodb-items.json",
@@ -137,7 +162,9 @@ public class DynamoDbService implements ResourceProvider {
new TypeReference<Map<String, ExportDescription>>() {}),
storageFactory.create("dynamodb", "dynamodb-imports.json",
new TypeReference<Map<String, ImportTableDescription>>() {}),
- regionResolver, streamService, kinesisForwarder, s3Service, objectMapper);
+ regionResolver, streamService, kinesisForwarder, s3Service, objectMapper,
+ config.services().dynamodb().vectorIndexAllocationSeconds(),
+ config.services().dynamodb().vectorIndexBackfillSeconds());
}
/** Package-private constructor for testing. */
@@ -172,6 +199,21 @@ public class DynamoDbService implements ResourceProvider {
KinesisStreamingForwarder kinesisForwarder,
S3Service s3Service,
ObjectMapper objectMapper) {
+ this(tableStore, itemStore, exportStore, importStore, regionResolver, streamService,
+ kinesisForwarder, s3Service, objectMapper, 0, 0);
+ }
+
+ DynamoDbService(StorageBackend<String, TableDefinition> tableStore,
+ StorageBackend<String, Map<String, JsonNode>> itemStore,
+ StorageBackend<String, ExportDescription> exportStore,
+ StorageBackend<String, ImportTableDescription> importStore,
+ RegionResolver regionResolver,
+ DynamoDbStreamService streamService,
+ KinesisStreamingForwarder kinesisForwarder,
+ S3Service s3Service,
+ ObjectMapper objectMapper,
+ int vectorIndexAllocationSeconds,
+ int vectorIndexBackfillSeconds) {
this.tableStore = tableStore;
this.itemStore = itemStore;
this.exportStore = exportStore;
@@ -181,6 +223,8 @@ public class DynamoDbService implements ResourceProvider {
this.kinesisForwarder = kinesisForwarder;
this.s3Service = s3Service;
this.objectMapper = objectMapper != null ? objectMapper : new ObjectMapper();
+ this.vectorIndexAllocationSeconds = vectorIndexAllocationSeconds;
+ this.vectorIndexBackfillSeconds = vectorIndexBackfillSeconds;
loadPersistedItems();
recoverInterruptedJobs();
}
@@ -261,6 +305,19 @@ public class DynamoDbService implements ResourceProvider {
List<GlobalSecondaryIndex> gsis,
List<LocalSecondaryIndex> lsis,
String region) {
+ return createTable(tableName, keySchema, attributeDefinitions, readCapacity, writeCapacity,
+ gsis, lsis, List.of(), null, region);
+ }
+
+ public TableDefinition createTable(String tableName,
+ List<KeySchemaElement> keySchema,
+ List<AttributeDefinition> attributeDefinitions,
+ Long readCapacity, Long writeCapacity,
+ List<GlobalSecondaryIndex> gsis,
+ List<LocalSecondaryIndex> lsis,
+ List<VectorIndexCreate> vectorIndexes,
+ String billingMode,
+ String region) {
// Enforce at the service boundary: CreateTable persists its input as the
// canonical table name and derives TableArn from it. An ARN-form input
// would produce ARN-on-ARN TableArn values. Handler-layer rejection alone
@@ -320,6 +377,10 @@ public class DynamoDbService implements ResourceProvider {
}
}
+ List<VectorIndexCreate> vectorIndexCreates = vectorIndexes != null ? vectorIndexes : List.of();
+ validateVectorIndexMembers(vectorIndexCreates);
+ validateVectorIndexes(vectorIndexCreates, List.of(), attributeDefinitions, billingMode);
+
Set<String> referencedAttrs = new HashSet<>();
keySchema.forEach(k -> referencedAttrs.add(k.getAttributeName()));
if (gsis != null) {
@@ -328,6 +389,9 @@ public class DynamoDbService implements ResourceProvider {
if (lsis != null) {
lsis.forEach(l -> l.getKeySchema().forEach(k -> referencedAttrs.add(k.getAttributeName())));
}
+ // A SearchSchema attribute counts as used, so the unused-definition rejection below does
+ // not fire for one that only a vector index reads.
+ vectorIndexCreates.forEach(v -> referencedAttrs.addAll(v.index().getSearchSchemaAttributeNames()));
Set<String> definedAttrs = attributeDefinitions == null
? Set.of()
: attributeDefinitions.stream()
@@ -372,6 +436,13 @@ public class DynamoDbService implements ResourceProvider {
}
}
}
+ for (VectorIndexCreate create : vectorIndexCreates) {
+ if (!indexNames.add(create.index().getIndexName())) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid: Duplicate index name: "
+ + create.index().getIndexName(), 400);
+ }
+ }
TableDefinition table = new TableDefinition(tableName, keySchema, attributeDefinitions,
region, regionResolver.getAccountId());
@@ -407,6 +478,19 @@ public class DynamoDbService implements ResourceProvider {
table.setLocalSecondaryIndexes(new ArrayList<>(lsis));
}
+ // A new VectorIndex is already ACTIVE with no creation timestamp, which is what AWS
+ // reports for one created with the table: Backfilling is never reported for it, and
+ // only the UpdateTable path runs the two timed phases.
+ if (!vectorIndexCreates.isEmpty()) {
+ List<VectorIndex> created = new ArrayList<>();
+ for (VectorIndexCreate create : vectorIndexCreates) {
+ VectorIndex vectorIndex = create.index();
+ vectorIndex.setIndexArn(table.getTableArn() + "/index/" + vectorIndex.getIndexName());
+ created.add(vectorIndex);
+ }
+ table.setVectorIndexes(created);
+ }
+
tableStore.put(storageKey, table);
itemsByTable.put(scopedItemsKey(storageKey), new ConcurrentSkipListMap<>());
LOG.infov("Created table: {0} in region {1}", tableName, region);
@@ -430,11 +514,128 @@ public class DynamoDbService implements ResourceProvider {
}
}
+ /**
+ * Validates each vector index a request adds against its own request model, the
+ * "N validation errors detected" family. AWS answers these before every other check on the
+ * request, including the online index limit an UpdateTable is held to.
+ *
+ * @param newIndexes the indexes the request adds, each with its request-model member path
+ */
+ private static void validateVectorIndexMembers(List<VectorIndexCreate> newIndexes) {
+ // The required members are reported before the length and range constraints.
+ for (VectorIndexCreate create : newIndexes) {
+ VectorIndex index = create.index();
+ String memberPath = create.memberPath();
+ requireVectorIndexMember(index.getDimensions(), memberPath + ".dimensions");
+ requireVectorIndexMember(index.getVectorAttributeName(), memberPath + ".vectorAttribute");
+ requireVectorIndexMember(index.getProjectionType(), memberPath + ".projection");
+ requireVectorIndexMember(index.getDistanceFunction(), memberPath + ".distanceFunction");
+ String indexName = index.getIndexName();
+ if (indexName == null || indexName.length() < MIN_VECTOR_INDEX_NAME_LENGTH) {
+ throw new AwsException("ValidationException",
+ "1 validation error detected: Value '" + indexName + "' at '"
+ + memberPath + ".indexName' failed to satisfy constraint: "
+ + "Member must have length greater than or equal to " + MIN_VECTOR_INDEX_NAME_LENGTH, 400);
+ }
+ String distanceFunction = index.getDistanceFunction();
+ if (!VECTOR_DISTANCE_FUNCTIONS.contains(distanceFunction)) {
+ throw new AwsException("ValidationException",
+ "1 validation error detected: Value '" + distanceFunction + "' at '"
+ + memberPath + ".distanceFunction' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: " + VECTOR_DISTANCE_FUNCTIONS, 400);
+ }
+ Long dimensions = index.getDimensions();
+ if (dimensions < 1) {
+ throw new AwsException("ValidationException",
+ "1 validation error detected: Value '" + dimensions + "' at '"
+ + memberPath + ".dimensions' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 1", 400);
+ }
+ }
+ }
+
+ /**
+ * Validates the vector indexes a request adds against the table they join, the
+ * "One or more parameter values were invalid" family. Runs after
+ * {@link #validateVectorIndexMembers}, so every index here carries its required members.
+ *
+ * @param newIndexes the indexes the request adds
+ * @param existingIndexes the indexes already on the table, empty on CreateTable
+ * @param attributeDefinitions every definition a SearchSchema element may resolve against
+ * @param billingMode the billing mode the table will have, null when the request
+ * leaves it at the PROVISIONED default
+ */
+ private static void validateVectorIndexes(List<VectorIndexCreate> newIndexes,
+ List<VectorIndex> existingIndexes,
+ List<AttributeDefinition> attributeDefinitions,
+ String billingMode) {
+ if (newIndexes.isEmpty()) {
+ return;
+ }
+
+ if (existingIndexes.size() + newIndexes.size() > MAX_VECTOR_INDEXES_PER_TABLE) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid: VectorIndex count exceeds "
+ + "the per-table limit of " + MAX_VECTOR_INDEXES_PER_TABLE, 400);
+ }
+
+ if (!"PAY_PER_REQUEST".equals(billingMode)) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid: Vector indexes are only supported "
+ + "for PAY_PER_REQUEST tables", 400);
+ }
+
+ Set<String> definedAttrs = attributeDefinitions == null
+ ? Set.of()
+ : attributeDefinitions.stream()
+ .map(AttributeDefinition::getAttributeName)
+ .collect(Collectors.toSet());
+
+ Map<String, Long> dimensionsByVectorAttribute = new HashMap<>();
+ for (VectorIndex existing : existingIndexes) {
+ dimensionsByVectorAttribute.put(existing.getVectorAttributeName(), existing.getDimensions());
+ }
+
+ for (VectorIndexCreate create : newIndexes) {
+ VectorIndex index = create.index();
+ Long dimensions = index.getDimensions();
+ if (dimensions > MAX_VECTOR_DIMENSIONS) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid: Number of dimensions must be "
+ + "between 1 and " + MAX_VECTOR_DIMENSIONS + " inclusive.", 400);
+ }
+ for (String searchSchemaAttr : index.getSearchSchemaAttributeNames()) {
+ if (!definedAttrs.contains(searchSchemaAttr)) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid: One element in SearchSchema "
+ + "is not defined in attribute definitions", 400);
+ }
+ }
+ String vectorAttribute = index.getVectorAttributeName();
+ Long seen = dimensionsByVectorAttribute.putIfAbsent(vectorAttribute, dimensions);
+ if (seen != null && !seen.equals(dimensions)) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid: Conflicting attribute definition "
+ + "for '" + vectorAttribute + "'. All VectorIndexes on the same vector "
+ + "attribute must use the same dimensions.", 400);
+ }
+ }
+ }
+
+ private static void requireVectorIndexMember(Object value, String memberPath) {
+ if (value == null) {
+ throw new AwsException("ValidationException",
+ "1 validation error detected: Value null at '" + memberPath
+ + "' failed to satisfy constraint: Member must not be null", 400);
+ }
+ }
+
public TableDefinition describeTable(String tableName, String region) {
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
TableDefinition table = tableStore.get(storageKey)
.orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ settleVectorIndexes(storageKey, table);
// Update dynamic counts
var items = itemsByTable.get(scopedItemsKey(storageKey));
@@ -444,6 +645,96 @@ public class DynamoDbService implements ResourceProvider {
return table;
}
+ /**
+ * Advances a vector index added by UpdateTable through its two phases on read, the way
+ * {@code AcmService} settles a pending certificate. Nothing else moves the index along, so
+ * a caller polling DescribeTable is what makes it progress.
+ *
+ * <p>The table leaves UPDATING at the allocation/backfill boundary, together with the switch
+ * from {@code Backfilling: false} to {@code Backfilling: true}, which is what AWS reports.
+ */
+ private void settleVectorIndexes(String storageKey, TableDefinition table) {
+ List<VectorIndex> indexes = table.getVectorIndexes();
+ if (indexes.isEmpty()) {
+ return;
+ }
+ Instant now = Instant.now();
+ boolean changed = false;
+ boolean anySettling = false;
+ boolean anyAllocating = false;
+ for (VectorIndex index : indexes) {
+ if (index.getCreationStartedAt() == null || !"CREATING".equals(index.getIndexStatus())) {
+ continue;
+ }
+ anySettling = true;
+ Instant backfillStart = index.getCreationStartedAt().plusSeconds(vectorIndexAllocationSeconds);
+ Instant activeAt = backfillStart.plusSeconds(vectorIndexBackfillSeconds);
+ if (!now.isBefore(activeAt)) {
+ index.setIndexStatus("ACTIVE");
+ index.setCreationStartedAt(null);
+ changed = true;
+ } else if (now.isBefore(backfillStart)) {
+ anyAllocating = true;
+ }
+ }
+ // Only a table this method is driving may leave UPDATING here: another caller's
+ // UPDATING is not this method's to clear.
+ if (anySettling && !anyAllocating && "UPDATING".equals(table.getTableStatus())) {
+ table.setTableStatus("ACTIVE");
+ changed = true;
+ }
+ if (changed) {
+ tableStore.put(storageKey, table);
+ }
+ }
+
+ /**
+ * Whether the index is past resource allocation and still building, which is the only state
+ * where AWS reports {@code Backfilling} at all. A settle must have run first.
+ */
+ public boolean isVectorIndexBackfilling(VectorIndex index) {
+ return reportsVectorIndexBackfilling(index) && !Instant.now().isBefore(
+ index.getCreationStartedAt().plusSeconds(vectorIndexAllocationSeconds));
+ }
+
+ /**
+ * Whether {@code Backfilling} is reported for this index at all. A vector index created with
+ * the table never reports it; one added by UpdateTable does until it reaches ACTIVE.
+ */
+ public boolean reportsVectorIndexBackfilling(VectorIndex index) {
+ return index.getCreationStartedAt() != null && "CREATING".equals(index.getIndexStatus());
+ }
+
+ /**
+ * The table a SearchVectors runs against, with its vector indexes settled. Skips the
+ * {@link #describeTable} item-count refresh, which is {@code O(items)} and which the
+ * SearchVectors response never reports.
+ */
+ public TableDefinition settledTable(String tableName, String region) {
+ String canonicalTableName = canonicalTableName(region, tableName);
+ String storageKey = regionKey(region, canonicalTableName);
+ TableDefinition table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ settleVectorIndexes(storageKey, table);
+ return table;
+ }
+
+ /**
+ * Every live item of the table, expired ones dropped. SearchVectors reads the whole table:
+ * it has no page boundary and no cursor, so it cannot go through {@link #scan}, which stops
+ * at the 1 MB response cap.
+ */
+ public List<JsonNode> liveItems(TableDefinition table, String region) {
+ String storageKey = regionKey(region, table.getTableName());
+ ConcurrentSkipListMap<String, JsonNode> items = itemsByTable.get(scopedItemsKey(storageKey));
+ if (items == null) {
+ return List.of();
+ }
+ return items.values().stream()
+ .filter(item -> !isExpired(item, table))
+ .toList();
+ }
+
/**
* The stored table definition without the {@link #describeTable} item-count refresh,
* which is {@code O(items)} on the item map. For callers that only need static metadata
@@ -518,6 +809,12 @@ public class DynamoDbService implements ResourceProvider {
var table = tableStore.get(storageKey)
.orElseThrow(() -> resourceNotFoundException(canonicalTableName));
requireNotCreating(table);
+ settleVectorIndexes(storageKey, table);
+ if (table.getVectorIndexes().stream().anyMatch(v -> "CREATING".equals(v.getIndexStatus()))) {
+ throw new AwsException("ResourceInUseException",
+ "Attempt to change a resource which is still in use: Cannot delete table while "
+ + "indexes are being created, updated, or deleted.", 400);
+ }
tableStore.delete(storageKey);
itemsByTable.remove(scopedItemsKey(storageKey));
itemLocks.remove(scopedItemsKey(storageKey));
@@ -1931,11 +2228,21 @@ public class DynamoDbService implements ResourceProvider {
public TableDefinition updateTable(String tableName, Long readCapacity, Long writeCapacity,
List<GlobalSecondaryIndex> gsiCreates, List<String> gsiDeletes,
List<AttributeDefinition> newAttrDefs, String region) {
+ return updateTable(tableName, readCapacity, writeCapacity, gsiCreates, gsiDeletes,
+ newAttrDefs, List.of(), List.of(), null, region);
+ }
+
+ public TableDefinition updateTable(String tableName, Long readCapacity, Long writeCapacity,
+ List<GlobalSecondaryIndex> gsiCreates, List<String> gsiDeletes,
+ List<AttributeDefinition> newAttrDefs,
+ List<VectorIndexCreate> vectorCreates, List<String> vectorDeletes,
+ String billingMode, String region) {
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
TableDefinition table = tableStore.get(storageKey)
.orElseThrow(() -> resourceNotFoundException(canonicalTableName));
requireNotCreating(table);
+ settleVectorIndexes(storageKey, table);
if (readCapacity != null && readCapacity <= 0) {
throw new AwsException("ValidationException",
@@ -1985,6 +2292,15 @@ public class DynamoDbService implements ResourceProvider {
}
}
+ List<VectorIndexCreate> vectorIndexCreates = vectorCreates != null ? vectorCreates : List.of();
+ List<String> vectorIndexDeletes = vectorDeletes != null ? vectorDeletes : List.of();
+ List<AttributeDefinition> knownAttrDefs = new ArrayList<>(table.getAttributeDefinitions());
+ if (newAttrDefs != null) {
+ knownAttrDefs.addAll(newAttrDefs);
+ }
+ validateVectorIndexUpdates(table, vectorIndexCreates, vectorIndexDeletes, knownAttrDefs,
+ billingMode != null ? billingMode : table.getBillingMode());
+
if (readCapacity != null) {
table.getProvisionedThroughput().setReadCapacityUnits(readCapacity);
}
@@ -2001,6 +2317,21 @@ public class DynamoDbService implements ResourceProvider {
table.getGlobalSecondaryIndexes().add(gsi);
}
+ for (String indexName : vectorIndexDeletes) {
+ table.getVectorIndexes().removeIf(v -> indexName.equals(v.getIndexName()));
+ }
+
+ for (VectorIndexCreate create : vectorIndexCreates) {
+ VectorIndex vectorIndex = create.index();
+ vectorIndex.setIndexArn(table.getTableArn() + "/index/" + vectorIndex.getIndexName());
+ vectorIndex.setIndexStatus("CREATING");
+ vectorIndex.setCreationStartedAt(Instant.now());
+ table.getVectorIndexes().add(vectorIndex);
+ }
+ if (!vectorIndexCreates.isEmpty()) {
+ table.setTableStatus("UPDATING");
+ }
+
if (newAttrDefs != null && !newAttrDefs.isEmpty()) {
List<AttributeDefinition> existing = table.getAttributeDefinitions();
for (AttributeDefinition newDef : newAttrDefs) {
@@ -2018,6 +2349,69 @@ public class DynamoDbService implements ResourceProvider {
return table;
}
+ /**
+ * Validates the {@code VectorIndexUpdates} of an UpdateTable request against the table's
+ * current lifecycle state.
+ *
+ * <p>AWS allows one online index operation per table at a time. That limit binds the table,
+ * not the request, so two creates in one request and a create issued while an earlier index
+ * still builds are refused the same way. Deleting an index that is still in its resource
+ * allocation phase is refused with a different error, and the same delete is accepted once
+ * the index reaches backfilling.
+ */
+ private void validateVectorIndexUpdates(TableDefinition table, List<VectorIndexCreate> creates,
+ List<String> deletes,
+ List<AttributeDefinition> attributeDefinitions,
+ String billingMode) {
+ validateVectorIndexMembers(creates);
+ // A table that holds a vector index may not leave PAY_PER_REQUEST, whether or not the
+ // request touches its indexes.
+ if (!table.getVectorIndexes().isEmpty() && !"PAY_PER_REQUEST".equals(billingMode)) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid: Vector indexes are only supported "
+ + "for PAY_PER_REQUEST tables", 400);
+ }
+ if (creates.isEmpty() && deletes.isEmpty()) {
+ return;
+ }
+ for (VectorIndexCreate create : creates) {
+ if (table.findVectorIndex(create.index().getIndexName()).isPresent()) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid: Duplicate index name: "
+ + create.index().getIndexName(), 400);
+ }
+ }
+ int deletesOfBuildingIndexes = 0;
+ for (String indexName : deletes) {
+ VectorIndex target = table.findVectorIndex(indexName)
+ .orElseThrow(() -> new AwsException("ValidationException",
+ "The table does not have the specified index: " + indexName, 400));
+ if (!"CREATING".equals(target.getIndexStatus())) {
+ continue;
+ }
+ if (!isVectorIndexBackfilling(target)) {
+ throw new AwsException("ResourceInUseException",
+ "Attempt to change a resource which is still in use: Index creation is in "
+ + "resource allocation phase. Retry deletion during backfilling phase or "
+ + "when the index is active. Table: " + table.getTableName()
+ + " Index: " + indexName, 400);
+ }
+ deletesOfBuildingIndexes++;
+ }
+ long building = table.getVectorIndexes().stream()
+ .filter(v -> "CREATING".equals(v.getIndexStatus()))
+ .count();
+ // Deleting an index that is still backfilling takes over the slot that index already
+ // holds, so it does not need one of its own.
+ long onlineOperations = building + creates.size() + deletes.size() - deletesOfBuildingIndexes;
+ if (onlineOperations > 1) {
+ throw new AwsException("LimitExceededException",
+ "Subscriber limit exceeded: Only 1 online index can be created or deleted "
+ + "simultaneously per table", 400);
+ }
+ validateVectorIndexes(creates, table.getVectorIndexes(), attributeDefinitions, billingMode);
+ }
+
/**
* UpdateTable keeps only the attribute definitions a key schema still uses. A definition sent
* with the request but used by no key is dropped in the same response, and deleting an index
@@ -2039,6 +2433,9 @@ public class DynamoDbService implements ResourceProvider {
used.add(k.getAttributeName());
}
}
+ for (VectorIndex vectorIndex : table.getVectorIndexes()) {
+ used.addAll(vectorIndex.getSearchSchemaAttributeNames());
+ }
List<AttributeDefinition> kept = table.getAttributeDefinitions().stream()
.filter(ad -> used.contains(ad.getAttributeName()))
.collect(Collectors.toCollection(ArrayList::new));
@@ -3609,6 +4006,57 @@ public class DynamoDbService implements ResourceProvider {
validateIndexKeySchema(table, item, lsi.getIndexName(), lsi.getKeySchema(), isUpdate);
}
}
+ for (VectorIndex vectorIndex : table.getVectorIndexes()) {
+ validateVectorIndexWrite(table, item, vectorIndex, isUpdate);
+ }
+ }
+
+ // A vector index constrains the shape of the vector attribute and, through its search
+ // schema, the HASH attribute it partitions on. An item carrying neither is written and
+ // left out of the index; one carrying a vector of the wrong shape is refused.
+ private void validateVectorIndexWrite(TableDefinition table, JsonNode item,
+ VectorIndex index, boolean isUpdate) {
+ JsonNode vector = item.get(index.getVectorAttributeName());
+ if (vector != null) {
+ validateVectorAttribute(vector, index);
+ }
+ String hashAttribute = index.getHashAttributeName();
+ if (hashAttribute != null) {
+ validateIndexKeySchema(table, item, index.getIndexName(),
+ List.of(new KeySchemaElement(hashAttribute, "HASH")), isUpdate);
+ }
+ }
+
+ // AWS punctuates these three inconsistently, which is reproduced here character for
+ // character: "Actual: S." carries a period and "Actual: 2" and "list" do not.
+ private void validateVectorAttribute(JsonNode vector, VectorIndex index) {
+ String attributeName = index.getVectorAttributeName();
+ String indexName = index.getIndexName();
+ validateAttributeValueShape(vector);
+ JsonNode components = vector.get("L");
+ if (components == null || !components.isArray()) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid. Invalid type for parameter "
+ + attributeName + ", Expected: 32-bit floating point number list IndexName: "
+ + indexName, 400);
+ }
+ Long dimensions = index.getDimensions();
+ if (components.size() != dimensions) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid. Invalid size for parameter "
+ + attributeName + ", Expected: " + dimensions
+ + ", Actual: " + components.size() + " IndexName: " + indexName, 400);
+ }
+ for (int i = 0; i < components.size(); i++) {
+ JsonNode component = components.get(i);
+ validateAttributeValueShape(component);
+ if (!component.has("N")) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid. Invalid type for parameter "
+ + attributeName + "[" + i + "], Expected: 32-bit floating point number, "
+ + "Actual: " + component.fieldNames().next() + ". IndexName: " + indexName, 400);
+ }
+ }
}
private void validateIndexKeySchema(TableDefinition table, JsonNode item,
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java
index b398356550..f7a6504349 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java
@@ -19,7 +19,7 @@ final class DynamoDbTransactCapacity {
static DynamoDbWriteCapacity.Cost write(TableDefinition table, JsonNode oldItem, JsonNode newItem) {
DynamoDbWriteCapacity.Cost cost = DynamoDbWriteCapacity.forWrite(table, oldItem, newItem);
- return new DynamoDbWriteCapacity.Cost(cost.table() * 2, cost.gsi(), cost.lsi());
+ return cost.withTable(cost.table() * 2);
}
static DynamoDbWriteCapacity.Cost conditionCheck(JsonNode item) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorScoring.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorScoring.java
new file mode 100644
index 0000000000..6132c1b2bd
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorScoring.java
@@ -0,0 +1,159 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+
+import java.math.BigDecimal;
+
+/**
+ * The numeric core of SearchVectors: the three distance functions and the f32 rendering of a
+ * stored vector.
+ *
+ * <p>AWS computes a score entirely in 32-bit floating point and sums the per-dimension terms with
+ * a pairwise binary-tree reduction. Measured on real DynamoDB in eu-west-2 (2026-09-23): the dot
+ * product of {@code [16777216,1,1,1,1,1,1,1]} with eight ones answers 16777222, and the same
+ * shape at sixteen dimensions answers 16777230. A double accumulator answers 16777224 and
+ * 16777232, a sequential f32 loop answers 16777216 twice, and a four-lane reduction answers
+ * 16777222 and 16777228, so only the tree reproduces both.
+ */
+final class DynamoDbVectorScoring {
+
+ // The band AWS writes in plain notation. Outside it the value comes back in scientific form.
+ private static final BigDecimal PLAIN_LOWER_BOUND = new BigDecimal("0.000001");
+ private static final BigDecimal PLAIN_UPPER_BOUND = new BigDecimal("1E+13");
+
+ static final String COSINE = "COSINE";
+ static final String EUCLIDEAN = "EUCLIDEAN";
+ static final String DOT_PRODUCT = "DOT_PRODUCT";
+
+ private DynamoDbVectorScoring() {}
+
+ /**
+ * Scores every candidate of one search against a fixed query vector.
+ *
+ * <p>Everything that depends only on the query is computed once here, and the per-dimension
+ * terms the tree sum consumes are written into a single buffer the scorer owns. A search
+ * scores its candidates one at a time, so that buffer needs no synchronization.
+ */
+ static final class Scorer {
+
+ private final String distanceFunction;
+ private final float[] query;
+ private final float[] terms;
+ private final double queryNorm;
+
+ Scorer(String distanceFunction, float[] query) {
+ this.distanceFunction = distanceFunction;
+ this.query = query;
+ this.terms = new float[query.length];
+ this.queryNorm = COSINE.equals(distanceFunction) ? Math.sqrt(selfDot(query, terms)) : 0.0;
+ }
+
+ /** The score of one stored vector against the query vector, widened for JSON. */
+ double score(float[] stored) {
+ return switch (distanceFunction) {
+ case COSINE -> cosine(stored);
+ case EUCLIDEAN -> euclidean(stored);
+ case DOT_PRODUCT -> dotProduct(query, stored, terms);
+ // CreateTable and UpdateTable reject an unknown value, so only a missing one
+ // reaches here: answer it rather than letting a null selector throw.
+ case null, default -> throw new AwsException("ValidationException",
+ "Unsupported distance function: " + distanceFunction, 400);
+ };
+ }
+
+ private float cosine(float[] stored) {
+ float dot = dotProduct(query, stored, terms);
+ double denominator = queryNorm * Math.sqrt(selfDot(stored, terms));
+ if (denominator == 0.0) {
+ // A zero vector has no direction. Scoring it 1 keeps the response valid JSON,
+ // which the NaN of a 0/0 division would not.
+ return 1.0f;
+ }
+ return (float) (1.0 - dot / denominator);
+ }
+
+ private float euclidean(float[] stored) {
+ for (int i = 0; i < query.length; i++) {
+ float difference = query[i] - stored[i];
+ terms[i] = difference * difference;
+ }
+ return (float) Math.sqrt(treeSum(terms));
+ }
+ }
+
+ /** Whether the higher score is the closer match, which only DOT_PRODUCT reports. */
+ static boolean higherIsCloser(String distanceFunction) {
+ return DOT_PRODUCT.equals(distanceFunction);
+ }
+
+ private static float dotProduct(float[] query, float[] stored, float[] terms) {
+ for (int i = 0; i < query.length; i++) {
+ terms[i] = query[i] * stored[i];
+ }
+ return treeSum(terms);
+ }
+
+ private static float selfDot(float[] vector, float[] terms) {
+ return dotProduct(vector, vector, terms);
+ }
+
+ /** Sums the terms by repeated pairwise halving, all arithmetic in float. Mutates {@code terms}. */
+ private static float treeSum(float[] terms) {
+ int n = terms.length;
+ while (n > 1) {
+ int half = (n + 1) / 2;
+ for (int i = 0; i + 1 < n; i += 2) {
+ terms[i / 2] = terms[i] + terms[i + 1];
+ }
+ if ((n & 1) == 1) {
+ terms[half - 1] = terms[n - 1];
+ }
+ n = half;
+ }
+ return n == 0 ? 0f : terms[0];
+ }
+
+ /**
+ * A stored f32 value as AWS writes it back: the shortest decimal naming that float. Plain
+ * notation with at least one fractional digit inside {@code [1e-6, 1e13)}, and
+ * {@code <mantissa>e<exponent>} outside it, with a lowercase e, no plus and no zero padding.
+ *
+ * <p>Both boundaries were measured on real DynamoDB (eu-west-2, 2026-09-23): 9.9e-7 comes back
+ * scientific and 1e-6 plain, 9e12 comes back plain and 1e13 scientific. 1 comes back as
+ * {@code 1.0} and 16777217 as {@code 16777216.0}.
+ *
+ * <p>{@code Float.toString} supplies the shortest round-tripping decimal, which it is for every
+ * normal float. Subnormals below {@link Float#MIN_NORMAL} are the one exception: AWS writes
+ * {@code Float.MIN_VALUE} as {@code 1e-45} where this renders {@code 1.4e-45}.
+ */
+ static String render(float value) {
+ if (value == 0f) {
+ return "0.0";
+ }
+ BigDecimal shortest = new BigDecimal(Float.toString(value)).stripTrailingZeros();
+ BigDecimal magnitude = shortest.abs();
+ if (magnitude.compareTo(PLAIN_LOWER_BOUND) >= 0 && magnitude.compareTo(PLAIN_UPPER_BOUND) < 0) {
+ String plain = shortest.toPlainString();
+ return plain.indexOf('.') < 0 ? plain + ".0" : plain;
+ }
+ return scientific(shortest);
+ }
+
+ /**
+ * The mantissa carries the shortest digits with no trailing {@code .0}, so 1e13 is written
+ * {@code 1e13} rather than {@code 1.0e13}.
+ */
+ private static String scientific(BigDecimal stripped) {
+ String digits = stripped.unscaledValue().abs().toString();
+ int exponent = digits.length() - 1 - stripped.scale();
+ StringBuilder out = new StringBuilder();
+ if (stripped.signum() < 0) {
+ out.append('-');
+ }
+ out.append(digits.charAt(0));
+ if (digits.length() > 1) {
+ out.append('.').append(digits, 1, digits.length());
+ }
+ return out.append('e').append(exponent).toString();
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorSearch.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorSearch.java
new file mode 100644
index 0000000000..9e3fa9b7e1
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorSearch.java
@@ -0,0 +1,293 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * The read path of a vector index: SearchConditionExpression validation, candidate filtering,
+ * scoring, top-K and result projection.
+ *
+ * <p>No vector index is materialized, the way no secondary index is materialized anywhere else in
+ * Floci. A search reads the table's items and drops the ones the index does not hold.
+ */
+final class DynamoDbVectorSearch {
+
+ private DynamoDbVectorSearch() {}
+
+ /** One search result: the projected item and its distance score. */
+ record Hit(ObjectNode item, double score) {}
+
+ /** A scored candidate, before the top-K cut decides whether it is worth projecting. */
+ private record Candidate(JsonNode item, double score) {}
+
+ static List<Hit> search(TableDefinition table, VectorIndex index, List<JsonNode> items,
+ JsonNode searchVector, int topK, String searchConditionExpression,
+ JsonNode exprAttrNames, JsonNode exprAttrValues,
+ String projectionExpression) {
+ float[] query = parseSearchVector(searchVector, index);
+ ExpressionEvaluator.Expr condition = parseSearchCondition(index, searchConditionExpression,
+ exprAttrNames, exprAttrValues);
+ ProjectionEvaluator.validateExpression(projectionExpression);
+
+ String hashAttribute = index.getHashAttributeName();
+ String distanceFunction = index.getDistanceFunction();
+ DynamoDbVectorScoring.Scorer scorer =
+ new DynamoDbVectorScoring.Scorer(distanceFunction, query);
+ List<Candidate> candidates = new ArrayList<>();
+ for (JsonNode item : items) {
+ if (hashAttribute != null && !item.hasNonNull(hashAttribute)) {
+ continue;
+ }
+ if (!ExpressionEvaluator.evaluate(condition, item, exprAttrNames, exprAttrValues)) {
+ continue;
+ }
+ float[] stored = storedVector(item, index);
+ if (stored == null) {
+ continue;
+ }
+ candidates.add(new Candidate(item, scorer.score(stored)));
+ }
+
+ Comparator<Candidate> byScore = Comparator.comparingDouble(Candidate::score);
+ candidates.sort(DynamoDbVectorScoring.higherIsCloser(distanceFunction)
+ ? byScore.reversed() : byScore);
+
+ List<Hit> hits = new ArrayList<>();
+ List<Candidate> top = candidates.subList(0, Math.min(topK, candidates.size()));
+ if (top.isEmpty()) {
+ return hits;
+ }
+ Set<String> projectedAttributes = projectedAttributeNames(table, index);
+ boolean rendersVector = projectionExpression != null && !projectionExpression.isBlank()
+ && ProjectionEvaluator.topLevelAttributes(projectionExpression, exprAttrNames)
+ .contains(index.getVectorAttributeName());
+ for (Candidate candidate : top) {
+ hits.add(new Hit(projectHit(candidate.item(), index, projectedAttributes,
+ projectionExpression, exprAttrNames, rendersVector), candidate.score()));
+ }
+ return hits;
+ }
+
+ // ── Search vector ──
+
+ private static float[] parseSearchVector(JsonNode searchVector, VectorIndex index) {
+ float[] query = searchVector != null && searchVector.isArray()
+ ? toVector(searchVector)
+ : new float[0];
+ if (query == null) {
+ throw invalidSearchVector();
+ }
+ long dimensions = index.getDimensions();
+ if (query.length != dimensions) {
+ throw new AwsException("ValidationException",
+ "Input search vector dimension " + query.length
+ + " does not match vector index dimension " + dimensions, 400);
+ }
+ return query;
+ }
+
+ private static AwsException invalidSearchVector() {
+ return new AwsException("ValidationException",
+ "Search vector contains invalid values. All values in the search vector must be a "
+ + "32-bit floating-point number attribute", 400);
+ }
+
+ // ── SearchConditionExpression ──
+
+ /**
+ * Parses and validates the condition, returning null when there is none.
+ *
+ * <p>Only equality is supported, on HASH and INLINE_FILTER elements alike. The comparator is
+ * checked on every clause before any membership check, because AWS answers {@code tenant < :t}
+ * on an index that declares {@code tenant} with the comparator message and {@code tenant = :t}
+ * on an index with no SearchSchema with the membership one.
+ */
+ private static ExpressionEvaluator.Expr parseSearchCondition(VectorIndex index, String expression,
+ JsonNode exprAttrNames,
+ JsonNode exprAttrValues) {
+ String hashAttribute = index.getHashAttributeName();
+ if (expression == null || expression.isBlank()) {
+ if (hashAttribute != null) {
+ throw missingHashCondition();
+ }
+ return null;
+ }
+ ExpressionEvaluator.validateExpression(expression, "SearchConditionExpression",
+ exprAttrNames, exprAttrValues);
+
+ ExpressionEvaluator.Expr condition = ExpressionEvaluator.parse(expression);
+ List<ExpressionEvaluator.CompareExpr> clauses = new ArrayList<>();
+ collectClauses(condition, clauses);
+ for (ExpressionEvaluator.CompareExpr clause : clauses) {
+ if (clause.op() != ExpressionEvaluator.TokenType.EQ) {
+ throw invalidComparator();
+ }
+ }
+
+ Set<String> schemaAttributes = new HashSet<>(index.getSearchSchemaAttributeNames());
+ boolean hashConstrained = false;
+ for (ExpressionEvaluator.CompareExpr clause : clauses) {
+ String attribute = conditionAttribute(clause, exprAttrNames);
+ if (!schemaAttributes.contains(attribute)) {
+ throw new AwsException("ValidationException",
+ "SearchConditionExpression must not contain any attributes that is not in "
+ + "SearchSchema. Invalid attribute: " + attribute, 400);
+ }
+ hashConstrained |= attribute.equals(hashAttribute);
+ }
+ if (hashAttribute != null && !hashConstrained) {
+ throw missingHashCondition();
+ }
+ return condition;
+ }
+
+ private static void collectClauses(ExpressionEvaluator.Expr expr,
+ List<ExpressionEvaluator.CompareExpr> clauses) {
+ if (expr instanceof ExpressionEvaluator.AndExpr and) {
+ for (ExpressionEvaluator.Expr operand : and.operands()) {
+ collectClauses(operand, clauses);
+ }
+ } else if (expr instanceof ExpressionEvaluator.CompareExpr compare) {
+ clauses.add(compare);
+ } else {
+ throw invalidComparator();
+ }
+ }
+
+ /** The attribute a clause constrains, with any {@code #alias} resolved. */
+ private static String conditionAttribute(ExpressionEvaluator.CompareExpr clause,
+ JsonNode exprAttrNames) {
+ ExpressionEvaluator.Operand operand = clause.left() instanceof ExpressionEvaluator.PathOperand
+ ? clause.left() : clause.right();
+ if (!(operand instanceof ExpressionEvaluator.PathOperand path)) {
+ throw invalidComparator();
+ }
+ StringBuilder name = new StringBuilder();
+ for (String segment : path.segments()) {
+ String resolved = segment.startsWith("#") && exprAttrNames != null
+ && exprAttrNames.hasNonNull(segment)
+ ? exprAttrNames.get(segment).asText() : segment;
+ if (!name.isEmpty() && !resolved.startsWith("[")) {
+ name.append('.');
+ }
+ name.append(resolved);
+ }
+ return name.toString();
+ }
+
+ private static AwsException missingHashCondition() {
+ return new AwsException("ValidationException",
+ "SearchConditionExpression must be provided when SearchSchema has a HASH key", 400);
+ }
+
+ private static AwsException invalidComparator() {
+ return new AwsException("ValidationException",
+ "Invalid SearchConditionExpression: Invalid comparator used in "
+ + "SearchConditionExpression", 400);
+ }
+
+ // ── Candidates ──
+
+ /**
+ * The item's vector narrowed to f32, or null when the item is not in the index at all. A
+ * missing attribute, a non-list value, a member that is not a number and a length other than
+ * the index dimension all leave the item out of the index rather than failing the search.
+ */
+ private static float[] storedVector(JsonNode item, VectorIndex index) {
+ JsonNode list = item.path(index.getVectorAttributeName()).path("L");
+ if (!list.isArray() || list.size() != index.getDimensions()) {
+ return null;
+ }
+ return toVector(list);
+ }
+
+ /**
+ * The list's members narrowed to f32, or null at the first member that is not a number
+ * attribute. The caller decides what that means: the search vector answers an error, a
+ * stored vector leaves its item out of the index.
+ */
+ private static float[] toVector(JsonNode list) {
+ float[] values = new float[list.size()];
+ for (int i = 0; i < values.length; i++) {
+ JsonNode number = list.get(i).path("N");
+ if (!number.isValueNode()) {
+ return null;
+ }
+ try {
+ values[i] = Float.parseFloat(number.asText());
+ } catch (NumberFormatException expected) {
+ // Not a number the index can hold, which the caller reports its own way.
+ return null;
+ }
+ }
+ return values;
+ }
+
+ // ── Projection ──
+
+ /**
+ * The attributes a hit carries. The index view is intersected with the ProjectionExpression
+ * when there is one, and the vector attribute is left out unless that expression names it.
+ */
+ private static ObjectNode projectHit(JsonNode item, VectorIndex index,
+ Set<String> projectedAttributes,
+ String projectionExpression, JsonNode exprAttrNames,
+ boolean rendersVector) {
+ ObjectNode view = "ALL".equals(index.getProjectionType())
+ ? shallowCopy((ObjectNode) item)
+ : ProjectionEvaluator.trimToAttributes((ObjectNode) item, projectedAttributes);
+ if (projectionExpression != null && !projectionExpression.isBlank()) {
+ if (rendersVector) {
+ view.set(index.getVectorAttributeName(),
+ renderVector(storedVector(item, index)));
+ }
+ return ProjectionEvaluator.project(view, projectionExpression, exprAttrNames);
+ }
+ view.remove(index.getVectorAttributeName());
+ return view;
+ }
+
+ /** A copy the projection can prune without touching the stored item. Values stay shared. */
+ private static ObjectNode shallowCopy(ObjectNode item) {
+ ObjectNode copy = JsonNodeFactory.instance.objectNode();
+ copy.setAll(item);
+ return copy;
+ }
+
+ private static Set<String> projectedAttributeNames(TableDefinition table, VectorIndex index) {
+ Set<String> projected = new HashSet<>();
+ for (KeySchemaElement key : table.getKeySchema()) {
+ projected.add(key.getAttributeName());
+ }
+ projected.addAll(index.getSearchSchemaAttributeNames());
+ if ("INCLUDE".equals(index.getProjectionType())) {
+ projected.addAll(index.getNonKeyAttributes());
+ }
+ return projected;
+ }
+
+ /** The index's own f32 copy of the vector, which is what a search returns. */
+ private static ObjectNode renderVector(float[] stored) {
+ ArrayNode list = JsonNodeFactory.instance.arrayNode();
+ for (float value : stored) {
+ ObjectNode element = JsonNodeFactory.instance.objectNode();
+ element.put("N", DynamoDbVectorScoring.render(value));
+ list.add(element);
+ }
+ ObjectNode wrapper = JsonNodeFactory.instance.objectNode();
+ wrapper.set("L", list);
+ return wrapper;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java
index 22884d99a3..c86e8d5ee4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacity.java
@@ -2,8 +2,10 @@ package io.github.hectorvent.floci.services.dynamodb;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@@ -17,15 +19,28 @@ import java.util.Map;
* stored view: one view write for an insert, an in-place update or a delete, two view
* writes when the index key changes (delete plus insert), and nothing when the stored
* view is unchanged or the item never touches the index.
+ *
+ * <p>A vector index is charged on the same delta, but reports bytes rather than units.
*/
final class DynamoDbWriteCapacity {
+ /** The smallest figure VectorWriteRequestBytes reports for an entry a write changes. */
+ private static final double VECTOR_WRITE_FLOOR_BYTES = 1024;
+
+ /** What the index stores per vector component, whatever the written decimal text costs. */
+ private static final int VECTOR_COMPONENT_BYTES = 4;
+
private DynamoDbWriteCapacity() {}
- record Cost(double table, Map<String, Double> gsi, Map<String, Double> lsi) {
+ record Cost(double table, Map<String, Double> gsi, Map<String, Double> lsi,
+ Map<String, Double> vectorBytes) {
private static final Cost ZERO = new Cost(0, Map.of(), Map.of());
+ Cost(double table, Map<String, Double> gsi, Map<String, Double> lsi) {
+ this(table, gsi, lsi, Map.of());
+ }
+
static Cost zero() {
return ZERO;
}
@@ -37,12 +52,19 @@ final class DynamoDbWriteCapacity {
return sum;
}
+ /** The same cost with the table arm replaced, which a transaction charges at twice the rate. */
+ Cost withTable(double units) {
+ return new Cost(units, gsi, lsi, vectorBytes);
+ }
+
Cost plus(Cost other) {
var mergedGsi = new LinkedHashMap<>(gsi);
other.gsi().forEach((name, units) -> mergedGsi.merge(name, units, Double::sum));
var mergedLsi = new LinkedHashMap<>(lsi);
other.lsi().forEach((name, units) -> mergedLsi.merge(name, units, Double::sum));
- return new Cost(table + other.table(), mergedGsi, mergedLsi);
+ LinkedHashMap<String, Double> mergedVector = new LinkedHashMap<>(vectorBytes);
+ other.vectorBytes().forEach((name, bytes) -> mergedVector.merge(name, bytes, Double::sum));
+ return new Cost(table + other.table(), mergedGsi, mergedLsi, mergedVector);
}
}
@@ -71,7 +93,83 @@ final class DynamoDbWriteCapacity {
}
}
}
- return new Cost(tableUnits, gsiUnits, lsiUnits);
+ LinkedHashMap<String, Double> vectorBytes = new LinkedHashMap<>();
+ for (VectorIndex vectorIndex : table.getVectorIndexes()) {
+ double bytes = vectorEntryBytes(vectorIndex,
+ vectorView(table, vectorIndex, oldItem),
+ vectorView(table, vectorIndex, newItem));
+ if (bytes > 0) {
+ vectorBytes.put(vectorIndex.getIndexName(), bytes);
+ }
+ }
+ return new Cost(tableUnits, gsiUnits, lsiUnits, vectorBytes);
+ }
+
+ private static double vectorEntryBytes(VectorIndex index, JsonNode oldView, JsonNode newView) {
+ if (oldView == null) {
+ return newView == null ? 0 : entryBytes(index, newView);
+ }
+ if (newView == null) {
+ return entryBytes(index, oldView);
+ }
+ if (oldView.equals(newView)) {
+ return 0;
+ }
+ return Math.max(entryBytes(index, oldView), entryBytes(index, newView));
+ }
+
+ /**
+ * The stored entry's size: every projected attribute at its item size, with the vector
+ * itself costing four bytes per component instead of the decimal text the item carries.
+ * The components are never read, only counted.
+ */
+ private static double entryBytes(VectorIndex index, JsonNode view) {
+ String vectorAttribute = index.getVectorAttributeName();
+ double bytes = DynamoDbItemSize.utf8Length(vectorAttribute)
+ + VECTOR_COMPONENT_BYTES * index.getDimensions();
+ for (Map.Entry<String, JsonNode> field : view.properties()) {
+ if (!vectorAttribute.equals(field.getKey())) {
+ bytes += DynamoDbItemSize.utf8Length(field.getKey())
+ + DynamoDbItemSize.attributeValueSize(field.getValue());
+ }
+ }
+ return Math.max(VECTOR_WRITE_FLOOR_BYTES, bytes);
+ }
+
+ /**
+ * What the vector index stores for the item, or null when the item is not in the index:
+ * it carries no vector attribute, or none for the search schema's HASH attribute. Both
+ * leave the item out silently rather than failing the write.
+ */
+ private static JsonNode vectorView(TableDefinition table, VectorIndex index, JsonNode item) {
+ if (item == null || !item.has(index.getVectorAttributeName())) {
+ return null;
+ }
+ String hashAttribute = index.getHashAttributeName();
+ if (hashAttribute != null && !item.has(hashAttribute)) {
+ return null;
+ }
+ String projectionType = index.getProjectionType();
+ if (projectionType == null || "ALL".equals(projectionType)) {
+ return item;
+ }
+ LinkedHashSet<String> keep = new LinkedHashSet<>();
+ keep.add(table.getPartitionKeyName());
+ if (table.getSortKeyName() != null) {
+ keep.add(table.getSortKeyName());
+ }
+ keep.add(index.getVectorAttributeName());
+ keep.addAll(index.getSearchSchemaAttributeNames());
+ if ("INCLUDE".equals(projectionType)) {
+ keep.addAll(index.getNonKeyAttributes());
+ }
+ ObjectNode view = JsonNodeFactory.instance.objectNode();
+ for (String name : keep) {
+ if (item.has(name)) {
+ view.set(name, item.get(name));
+ }
+ }
+ return view;
}
private static double indexUnits(TableDefinition table, List<KeySchemaElement> keySchema,
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/SearchSchemaElement.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/SearchSchemaElement.java
new file mode 100644
index 0000000000..f061ac37c6
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/SearchSchemaElement.java
@@ -0,0 +1,27 @@
+package io.github.hectorvent.floci.services.dynamodb.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class SearchSchemaElement {
+
+ private String attributeName;
+ private String searchSchemaElementType; // HASH or INLINE_FILTER
+
+ public SearchSchemaElement() {}
+
+ public SearchSchemaElement(String attributeName, String searchSchemaElementType) {
+ this.attributeName = attributeName;
+ this.searchSchemaElementType = searchSchemaElementType;
+ }
+
+ public String getAttributeName() { return attributeName; }
+ public void setAttributeName(String attributeName) { this.attributeName = attributeName; }
+
+ public String getSearchSchemaElementType() { return searchSchemaElementType; }
+ public void setSearchSchemaElementType(String searchSchemaElementType) {
+ this.searchSchemaElementType = searchSchemaElementType;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableDefinition.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableDefinition.java
index ba21da8284..1c057ba68f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableDefinition.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableDefinition.java
@@ -31,6 +31,7 @@ public class TableDefinition {
private Map<String, String> tags;
private List<GlobalSecondaryIndex> globalSecondaryIndexes;
private List<LocalSecondaryIndex> localSecondaryIndexes;
+ private List<VectorIndex> vectorIndexes;
private String billingMode; // "PROVISIONED" or "PAY_PER_REQUEST"
private String ttlAttributeName;
private boolean ttlEnabled;
@@ -64,6 +65,7 @@ public class TableDefinition {
this.tags = new HashMap<>();
this.globalSecondaryIndexes = new ArrayList<>();
this.localSecondaryIndexes = new ArrayList<>();
+ this.vectorIndexes = new ArrayList<>();
this.pointInTimeRecoveryRecoveryPeriodInDays = 35;
this.kinesisStreamingDestinations = new ArrayList<>();
this.replicaRegions = new ArrayList<>();
@@ -92,6 +94,7 @@ public class TableDefinition {
this.tags = new HashMap<>();
this.globalSecondaryIndexes = new ArrayList<>();
this.localSecondaryIndexes = new ArrayList<>();
+ this.vectorIndexes = new ArrayList<>();
this.pointInTimeRecoveryRecoveryPeriodInDays = 35;
this.kinesisStreamingDestinations = new ArrayList<>();
this.replicaRegions = new ArrayList<>();
@@ -137,6 +140,16 @@ public class TableDefinition {
this.localSecondaryIndexes = localSecondaryIndexes != null ? localSecondaryIndexes : new ArrayList<>();
}
+ public List<VectorIndex> getVectorIndexes() {
+ if (vectorIndexes == null) {
+ vectorIndexes = new ArrayList<>();
+ }
+ return vectorIndexes;
+ }
+ public void setVectorIndexes(List<VectorIndex> vectorIndexes) {
+ this.vectorIndexes = vectorIndexes != null ? vectorIndexes : new ArrayList<>();
+ }
+
public String getBillingMode() { return billingMode; }
public void setBillingMode(String billingMode) { this.billingMode = billingMode; }
@@ -279,4 +292,14 @@ public class TableDefinition {
.filter(l -> indexName.equals(l.getIndexName()))
.findFirst();
}
+
+ @JsonIgnore
+ public Optional<VectorIndex> findVectorIndex(String indexName) {
+ if (vectorIndexes == null || vectorIndexes.isEmpty() || indexName == null) {
+ return Optional.empty();
+ }
+ return vectorIndexes.stream()
+ .filter(v -> indexName.equals(v.getIndexName()))
+ .findFirst();
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/VectorIndex.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/VectorIndex.java
new file mode 100644
index 0000000000..493fd02d43
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/VectorIndex.java
@@ -0,0 +1,111 @@
+package io.github.hectorvent.floci.services.dynamodb.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class VectorIndex {
+
+ private static final String HASH = "HASH";
+
+ private String indexName;
+ private String vectorAttributeName;
+ private List<SearchSchemaElement> searchSchema;
+ private String projectionType;
+ private List<String> nonKeyAttributes;
+ private Long dimensions;
+ private String distanceFunction;
+ private String indexArn;
+ private String indexStatus = "ACTIVE";
+ // Set only when the index was added by UpdateTable, which reports the allocation and
+ // backfilling phases; null on the CreateTable path, which reports an ACTIVE index at once.
+ private Instant creationStartedAt;
+
+ public VectorIndex() {
+ this.searchSchema = new ArrayList<>();
+ this.nonKeyAttributes = new ArrayList<>();
+ }
+
+ public VectorIndex(String indexName, String vectorAttributeName,
+ List<SearchSchemaElement> searchSchema, String projectionType,
+ List<String> nonKeyAttributes, Long dimensions, String distanceFunction) {
+ this.indexName = indexName;
+ this.vectorAttributeName = vectorAttributeName;
+ this.searchSchema = searchSchema != null ? searchSchema : new ArrayList<>();
+ // Left null when the request omitted Projection, which is a required member the
+ // CreateTable and UpdateTable validation rejects.
+ this.projectionType = projectionType;
+ this.nonKeyAttributes = nonKeyAttributes != null ? nonKeyAttributes : new ArrayList<>();
+ this.dimensions = dimensions;
+ this.distanceFunction = distanceFunction;
+ }
+
+ public String getIndexName() { return indexName; }
+ public void setIndexName(String indexName) { this.indexName = indexName; }
+
+ public String getVectorAttributeName() { return vectorAttributeName; }
+ public void setVectorAttributeName(String vectorAttributeName) { this.vectorAttributeName = vectorAttributeName; }
+
+ public List<SearchSchemaElement> getSearchSchema() { return searchSchema; }
+ public void setSearchSchema(List<SearchSchemaElement> searchSchema) {
+ this.searchSchema = searchSchema != null ? searchSchema : new ArrayList<>();
+ }
+
+ public String getProjectionType() { return projectionType; }
+ public void setProjectionType(String projectionType) { this.projectionType = projectionType; }
+
+ public List<String> getNonKeyAttributes() { return nonKeyAttributes; }
+ public void setNonKeyAttributes(List<String> nonKeyAttributes) {
+ this.nonKeyAttributes = nonKeyAttributes != null ? nonKeyAttributes : new ArrayList<>();
+ }
+
+ public Long getDimensions() { return dimensions; }
+ public void setDimensions(Long dimensions) { this.dimensions = dimensions; }
+
+ public String getDistanceFunction() { return distanceFunction; }
+ public void setDistanceFunction(String distanceFunction) { this.distanceFunction = distanceFunction; }
+
+ public String getIndexArn() { return indexArn; }
+ public void setIndexArn(String indexArn) { this.indexArn = indexArn; }
+
+ public String getIndexStatus() { return indexStatus; }
+ public void setIndexStatus(String indexStatus) { this.indexStatus = indexStatus; }
+
+ public Instant getCreationStartedAt() { return creationStartedAt; }
+ public void setCreationStartedAt(Instant creationStartedAt) { this.creationStartedAt = creationStartedAt; }
+
+ /**
+ * The attribute names the index filters on, in search-schema order.
+ *
+ * <p>{@code @JsonIgnore}d for the reason {@code TableDefinition.getSortKeyNames()} documents:
+ * it is derived from {@code searchSchema}, and without a backing setter Jackson's
+ * getter-as-setter fallback appends into the immutable list this method returns.
+ */
+ @JsonIgnore
+ public List<String> getSearchSchemaAttributeNames() {
+ return searchSchema.stream()
+ .map(SearchSchemaElement::getAttributeName)
+ .toList();
+ }
+
+ /**
+ * The attribute the search schema partitions on, or null when the index declares no HASH
+ * element. {@code @JsonIgnore}d for the same reason as
+ * {@link #getSearchSchemaAttributeNames()}: it is derived from {@code searchSchema}.
+ */
+ @JsonIgnore
+ public String getHashAttributeName() {
+ for (SearchSchemaElement element : searchSchema) {
+ if (HASH.equals(element.getSearchSchemaElementType())) {
+ return element.getAttributeName();
+ }
+ }
+ return null;
+ }
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 6d883ea9b5..aef6239a21 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -303,6 +303,8 @@ floci:
global-bucket-namespace: false
dynamodb:
enabled: true
+ vector-index-allocation-seconds: 4 # FLOCI_SERVICES_DYNAMODB_VECTOR_INDEX_ALLOCATION_SECONDS. Resource allocation phase of a vector index added by UpdateTable.
+ vector-index-backfill-seconds: 10 # FLOCI_SERVICES_DYNAMODB_VECTOR_INDEX_BACKFILL_SECONDS. Backfilling phase that follows it.
sns:
enabled: true
lambda:
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorIndexLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorIndexLifecycleTest.java
new file mode 100644
index 0000000000..e8b7fc5a27
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorIndexLifecycleTest.java
@@ -0,0 +1,215 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The two phases a vector index added by UpdateTable walks through.
+ *
+ * <p>The phases are derived from one stored instant on read, so a test moves through them by
+ * rewinding that instant rather than by sleeping, the way {@code AcmIssuanceServiceTest} rewinds
+ * a certificate's validation wait. Both durations are 0 under {@code @QuarkusTest}, which is why
+ * this coverage lives in a unit test.
+ */
+class DynamoDbVectorIndexLifecycleTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String TABLE = "Docs";
+ private static final int ALLOCATION_SECONDS = 4;
+ private static final int BACKFILL_SECONDS = 10;
+
+ private DynamoDbService service;
+
+ @BeforeEach
+ void setUp() {
+ service = new DynamoDbService(new InMemoryStorage<>(), null, null, null,
+ new RegionResolver(REGION, "000000000000"), null, null, null, null,
+ ALLOCATION_SECONDS, BACKFILL_SECONDS);
+ TableDefinition table = service.createTable(TABLE,
+ List.of(new KeySchemaElement("pk", "HASH")),
+ List.of(new AttributeDefinition("pk", "S")),
+ null, null, List.of(), List.of(), List.of(), "PAY_PER_REQUEST", REGION);
+ // The handler stamps the billing mode after CreateTable returns, and UpdateTable reads it
+ // back whenever the request does not resend BillingMode.
+ table.setBillingMode("PAY_PER_REQUEST");
+ }
+
+ private static VectorIndex vectorIndex(String indexName) {
+ return new VectorIndex(indexName, "embedding", List.of(), "ALL", List.of(), 3L, "COSINE");
+ }
+
+ private static DynamoDbService.VectorIndexCreate create(String indexName, String memberPath) {
+ return new DynamoDbService.VectorIndexCreate(vectorIndex(indexName), memberPath);
+ }
+
+ private TableDefinition addIndex(String indexName) {
+ return service.updateTable(TABLE, null, null, List.of(), List.of(), List.of(),
+ List.of(create(indexName, "vectorIndexUpdates.1.member.create")), List.of(), null, REGION);
+ }
+
+ private TableDefinition deleteIndex(String indexName) {
+ return service.updateTable(TABLE, null, null, List.of(), List.of(), List.of(),
+ List.of(), List.of(indexName), null, REGION);
+ }
+
+ private void elapse(String indexName, int seconds) {
+ service.describeTable(TABLE, REGION).findVectorIndex(indexName).orElseThrow()
+ .setCreationStartedAt(Instant.now().minusSeconds(seconds));
+ }
+
+ private VectorIndex describeIndex(String indexName) {
+ return service.describeTable(TABLE, REGION).findVectorIndex(indexName).orElseThrow();
+ }
+
+ @Test
+ void indexCreatedWithTheTableIsActiveAndNeverReportsBackfilling() {
+ service.createTable("Fresh",
+ List.of(new KeySchemaElement("pk", "HASH")),
+ List.of(new AttributeDefinition("pk", "S")),
+ null, null, List.of(), List.of(), List.of(create("vix", "vectorIndexes.1.member")),
+ "PAY_PER_REQUEST", REGION);
+
+ TableDefinition described = service.describeTable("Fresh", REGION);
+ VectorIndex index = described.findVectorIndex("vix").orElseThrow();
+
+ assertEquals("ACTIVE", described.getTableStatus());
+ assertEquals("ACTIVE", index.getIndexStatus());
+ assertFalse(service.reportsVectorIndexBackfilling(index));
+ }
+
+ @Test
+ void resourceAllocationReportsAnUpdatingTableAndBackfillingFalse() {
+ assertEquals("UPDATING", addIndex("vix").getTableStatus());
+
+ TableDefinition described = service.describeTable(TABLE, REGION);
+ VectorIndex index = described.findVectorIndex("vix").orElseThrow();
+
+ assertEquals("UPDATING", described.getTableStatus());
+ assertEquals("CREATING", index.getIndexStatus());
+ assertTrue(service.reportsVectorIndexBackfilling(index));
+ assertFalse(service.isVectorIndexBackfilling(index));
+ }
+
+ @Test
+ void deleteDuringResourceAllocationIsRefused() {
+ addIndex("vix");
+
+ AwsException refused = assertThrows(AwsException.class, () -> deleteIndex("vix"));
+
+ assertEquals("ResourceInUseException", refused.getErrorCode());
+ assertEquals("Attempt to change a resource which is still in use: Index creation is in "
+ + "resource allocation phase. Retry deletion during backfilling phase or when "
+ + "the index is active. Table: " + TABLE + " Index: vix", refused.getMessage());
+ assertEquals("CREATING", describeIndex("vix").getIndexStatus());
+ }
+
+ @Test
+ void backfillingReportsAnActiveTableAndACreatingIndex() {
+ addIndex("vix");
+ elapse("vix", ALLOCATION_SECONDS + 1);
+
+ TableDefinition described = service.describeTable(TABLE, REGION);
+ VectorIndex index = described.findVectorIndex("vix").orElseThrow();
+
+ assertEquals("ACTIVE", described.getTableStatus());
+ assertEquals("CREATING", index.getIndexStatus());
+ assertTrue(service.isVectorIndexBackfilling(index));
+ }
+
+ @Test
+ void deleteDuringBackfillingSucceeds() {
+ addIndex("vix");
+ elapse("vix", ALLOCATION_SECONDS + 1);
+
+ assertTrue(deleteIndex("vix").getVectorIndexes().isEmpty());
+ }
+
+ @Test
+ void indexEndsActiveWithBackfillingGone() {
+ addIndex("vix");
+ elapse("vix", ALLOCATION_SECONDS + BACKFILL_SECONDS + 1);
+
+ TableDefinition described = service.describeTable(TABLE, REGION);
+ VectorIndex index = described.findVectorIndex("vix").orElseThrow();
+
+ assertEquals("ACTIVE", described.getTableStatus());
+ assertEquals("ACTIVE", index.getIndexStatus());
+ assertFalse(service.reportsVectorIndexBackfilling(index));
+ }
+
+ @Test
+ void twoCreatesInOneRequestExceedTheOnlineIndexLimit() {
+ AwsException refused = assertThrows(AwsException.class, () ->
+ service.updateTable(TABLE, null, null, List.of(), List.of(), List.of(),
+ List.of(create("one", "vectorIndexUpdates.1.member.create"),
+ create("two", "vectorIndexUpdates.2.member.create")),
+ List.of(), null, REGION));
+
+ assertEquals("LimitExceededException", refused.getErrorCode());
+ assertEquals("Subscriber limit exceeded: Only 1 online index can be created or deleted "
+ + "simultaneously per table", refused.getMessage());
+ assertTrue(service.describeTable(TABLE, REGION).getVectorIndexes().isEmpty());
+ }
+
+ @Test
+ void secondCreateWhileAnIndexBuildsExceedsTheOnlineIndexLimit() {
+ addIndex("first");
+
+ AwsException refused = assertThrows(AwsException.class, () -> addIndex("second"));
+
+ assertEquals("LimitExceededException", refused.getErrorCode());
+ assertEquals("Subscriber limit exceeded: Only 1 online index can be created or deleted "
+ + "simultaneously per table", refused.getMessage());
+ assertEquals(1, service.describeTable(TABLE, REGION).getVectorIndexes().size());
+ }
+
+ @Test
+ void anotherIndexMayBeCreatedOnceTheFirstIsActive() {
+ addIndex("first");
+ elapse("first", ALLOCATION_SECONDS + BACKFILL_SECONDS + 1);
+ service.describeTable(TABLE, REGION);
+
+ addIndex("second");
+
+ assertEquals(List.of("first", "second"),
+ service.describeTable(TABLE, REGION).getVectorIndexes().stream()
+ .map(VectorIndex::getIndexName).toList());
+ }
+
+ @Test
+ void deleteTableIsRefusedWhileAVectorIndexBuilds() {
+ addIndex("vix");
+
+ AwsException refused = assertThrows(AwsException.class,
+ () -> service.deleteTable(TABLE, REGION));
+
+ assertEquals("ResourceInUseException", refused.getErrorCode());
+ assertEquals("Attempt to change a resource which is still in use: Cannot delete table "
+ + "while indexes are being created, updated, or deleted.", refused.getMessage());
+ }
+
+ @Test
+ void deleteTableSucceedsOnceTheIndexIsActive() {
+ addIndex("vix");
+ elapse("vix", ALLOCATION_SECONDS + BACKFILL_SECONDS + 1);
+
+ service.deleteTable(TABLE, REGION);
+
+ assertThrows(AwsException.class, () -> service.describeTable(TABLE, REGION));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorScoringTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorScoringTest.java
new file mode 100644
index 0000000000..9a4f9a15eb
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorScoringTest.java
@@ -0,0 +1,109 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * Every number below was measured on real DynamoDB in eu-west-2 on 2026-09-23.
+ */
+class DynamoDbVectorScoringTest {
+
+ private static final float[] QUERY = {1f, 0f, 0f};
+
+ @Test
+ void scoresTheIdenticalVector() {
+ float[] stored = {1f, 0f, 0f};
+ assertEquals(0.0, score("COSINE", QUERY, stored));
+ assertEquals(0.0, score("EUCLIDEAN", QUERY, stored));
+ assertEquals(1.0, score("DOT_PRODUCT", QUERY, stored));
+ }
+
+ @Test
+ void scoresAUnitNormMix() {
+ float[] stored = {0.6f, 0.8f, 0f};
+ assertEquals(0.3999999761581421, score("COSINE", QUERY, stored));
+ assertEquals(0.8944271802902222, score("EUCLIDEAN", QUERY, stored));
+ assertEquals(0.6000000238418579, score("DOT_PRODUCT", QUERY, stored));
+ }
+
+ @Test
+ void scoresBeyondF32Precision() {
+ float[] stored = {0.1f, 16777217f, 1.000000059604644775390625f};
+ assertEquals(1.0, score("COSINE", QUERY, stored));
+ assertEquals(16777216.0, score("EUCLIDEAN", QUERY, stored));
+ assertEquals(0.10000000149011612, score("DOT_PRODUCT", QUERY, stored));
+ }
+
+ /**
+ * The exact sum is 16777223, which needs more than 24 bits. A double accumulator answers
+ * 16777224, a sequential f32 loop answers 16777216 and a four-lane reduction answers
+ * 16777222, so this case alone does not separate the last two.
+ */
+ @Test
+ void sumsEightDimensionsAsAPairwiseTree() {
+ float[] query = ones(8);
+ float[] stored = ones(8);
+ stored[0] = 16777216f;
+ assertEquals(16777222f, score("DOT_PRODUCT", query, stored));
+ }
+
+ /** At sixteen dimensions the four-lane reduction answers 16777228 and the tree 16777230. */
+ @Test
+ void sumsSixteenDimensionsAsAPairwiseTree() {
+ float[] query = ones(16);
+ float[] stored = ones(16);
+ stored[0] = 16777216f;
+ assertEquals(16777230f, score("DOT_PRODUCT", query, stored));
+ }
+
+ @Test
+ void dotProductIsTheOnlyFunctionWhereHigherIsCloser() {
+ assertTrue(DynamoDbVectorScoring.higherIsCloser("DOT_PRODUCT"));
+ assertFalse(DynamoDbVectorScoring.higherIsCloser("COSINE"));
+ assertFalse(DynamoDbVectorScoring.higherIsCloser("EUCLIDEAN"));
+ }
+
+ @Test
+ void rendersTheStoredFloatAsShortestPlainDecimal() {
+ assertEquals("1.0", DynamoDbVectorScoring.render(1f));
+ assertEquals("0.0", DynamoDbVectorScoring.render(0f));
+ assertEquals("0.6", DynamoDbVectorScoring.render(0.6f));
+ assertEquals("0.8", DynamoDbVectorScoring.render(0.8f));
+ assertEquals("0.1", DynamoDbVectorScoring.render(0.1f));
+ assertEquals("16777216.0", DynamoDbVectorScoring.render(16777217f));
+ assertEquals("1.0", DynamoDbVectorScoring.render(1.000000059604644775390625f));
+ assertEquals("7.0", DynamoDbVectorScoring.render(7f));
+ assertEquals("-0.5", DynamoDbVectorScoring.render(-0.5f));
+ assertEquals("0.0", DynamoDbVectorScoring.render(-0f));
+ assertEquals("0.000123", DynamoDbVectorScoring.render(0.000123f));
+ assertEquals("1234568000.0", DynamoDbVectorScoring.render(1234567900f));
+ assertEquals("-10000000000.0", DynamoDbVectorScoring.render(-1e10f));
+ }
+
+ @Test
+ void rendersOutsideThePlainBandInScientificNotation() {
+ assertEquals("0.000001", DynamoDbVectorScoring.render(1e-6f));
+ assertEquals("0.0000011", DynamoDbVectorScoring.render(1.1e-6f));
+ assertEquals("9.9e-7", DynamoDbVectorScoring.render(9.9e-7f));
+ assertEquals("1e-7", DynamoDbVectorScoring.render(1e-7f));
+ assertEquals("1.25e-7", DynamoDbVectorScoring.render(1.25e-7f));
+ assertEquals("9000000000000.0", DynamoDbVectorScoring.render(9e12f));
+ assertEquals("1e13", DynamoDbVectorScoring.render(1e13f));
+ assertEquals("1.5e13", DynamoDbVectorScoring.render(1.5e13f));
+ assertEquals("1e30", DynamoDbVectorScoring.render(1e30f));
+ assertEquals("3.4e38", DynamoDbVectorScoring.render(3.4e38f));
+ }
+
+ private static double score(String distanceFunction, float[] query, float[] stored) {
+ return new DynamoDbVectorScoring.Scorer(distanceFunction, query).score(stored);
+ }
+
+ private static float[] ones(int dimensions) {
+ float[] vector = new float[dimensions];
+ Arrays.fill(vector, 1f);
+ return vector;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorSearchIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorSearchIntegrationTest.java
new file mode 100644
index 0000000000..acd52794df
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbVectorSearchIntegrationTest.java
@@ -0,0 +1,981 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.RestAssured;
+import io.restassured.response.ValidatableResponse;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The vector index surface over the wire: DescribeTable round-tripping, the three distance
+ * functions, projection, and the exact wording of every rejected request.
+ *
+ * <p>Both phase durations are 0 in the test configuration, so an index added by UpdateTable is
+ * ACTIVE at once here. {@link DynamoDbVectorIndexLifecycleTest} covers the two phases.
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class DynamoDbVectorSearchIntegrationTest {
+
+ private static final String CT = "application/x-amz-json-1.0";
+ private static final String TARGET = "DynamoDB_20120810.";
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private static final String DOCS = "VectorSearchDocs";
+ private static final String PLAIN = "VectorPlainIndex";
+ private static final String SCHEMA = "VectorSchemaIndex";
+ private static final String REJECTED = "VectorRejectedTable";
+
+ private static final String QUERY_VECTOR = """
+ [{"N": "1"}, {"N": "0"}, {"N": "0"}]""";
+
+ private static int testPort;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ // @AfterAll runs after Quarkus has reset the shared port, so it needs its own copy. Taken
+ // per test rather than in createTables, which a filtered single-method run never reaches.
+ @BeforeEach
+ void capturePort() {
+ testPort = RestAssured.port;
+ }
+
+ @AfterAll
+ static void cleanup() {
+ for (String table : List.of(DOCS, PLAIN, SCHEMA)) {
+ given()
+ .port(testPort)
+ .header("X-Amz-Target", TARGET + "DeleteTable")
+ .contentType(CT)
+ .body("""
+ {"TableName": "%s"}
+ """.formatted(table))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+ }
+
+ private static ValidatableResponse call(String action, String body) {
+ return given()
+ .header("X-Amz-Target", TARGET + action)
+ .contentType(CT)
+ .body(body)
+ .when()
+ .post("/")
+ .then();
+ }
+
+ private static JsonNode json(String action, String body) throws Exception {
+ return MAPPER.readTree(call(action, body).statusCode(200).extract().asString());
+ }
+
+ private static void expectValidation(String action, String body, String message) {
+ call(action, body)
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"))
+ .body("message", equalTo(message));
+ }
+
+ /** A SearchVectors body for the shared query vector, plus any further members verbatim. */
+ private static String search(String table, String index, int topK, String extraMembers) {
+ return """
+ {
+ "TableName": "%s",
+ "IndexName": "%s",
+ "SearchVector": %s,
+ "TopK": %d%s
+ }
+ """.formatted(table, index, QUERY_VECTOR, topK, extraMembers);
+ }
+
+ private static List<String> keysOf(JsonNode response) {
+ List<String> keys = new ArrayList<>();
+ for (JsonNode result : response.path("SearchResults")) {
+ keys.add(result.path("Item").path("pk").path("S").asText());
+ }
+ return keys;
+ }
+
+ private static List<Double> scoresOf(JsonNode response) {
+ List<Double> scores = new ArrayList<>();
+ for (JsonNode result : response.path("SearchResults")) {
+ scores.add(result.path("Score").asDouble());
+ }
+ return scores;
+ }
+
+ private static List<String> vectorOf(JsonNode searchResult) {
+ List<String> components = new ArrayList<>();
+ for (JsonNode component : searchResult.path("Item").path("embedding").path("L")) {
+ components.add(component.path("N").asText());
+ }
+ return components;
+ }
+
+ @Test
+ @Order(1)
+ void createTables() {
+ call("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "cosine",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"},
+ {"IndexName": "euclid",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "EUCLIDEAN"},
+ {"IndexName": "dotp",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "DOT_PRODUCT"}
+ ]
+ }
+ """.formatted(DOCS))
+ .statusCode(200)
+ .body("TableDescription.VectorIndexes.size()", equalTo(3))
+ .body("TableDescription.VectorIndexes.IndexName",
+ contains("cosine", "euclid", "dotp"))
+ .body("TableDescription.VectorIndexes.IndexStatus",
+ contains("ACTIVE", "ACTIVE", "ACTIVE"));
+
+ call("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "plain",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}
+ ]
+ }
+ """.formatted(PLAIN))
+ .statusCode(200);
+
+ call("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [
+ {"AttributeName": "pk", "AttributeType": "S"},
+ {"AttributeName": "tenant", "AttributeType": "S"},
+ {"AttributeName": "category", "AttributeType": "S"}
+ ],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "schema",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "SearchSchema": [
+ {"AttributeName": "tenant", "SearchSchemaElementType": "HASH"},
+ {"AttributeName": "category", "SearchSchemaElementType": "INLINE_FILTER"}
+ ],
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}
+ ]
+ }
+ """.formatted(SCHEMA))
+ .statusCode(200);
+ }
+
+ @Test
+ @Order(2)
+ void seedItems() {
+ putVector("a", "1", "0", "0");
+ putVector("b", "0", "1", "0");
+ putVector("c", "-1", "0", "0");
+ putVector("d", "0.6", "0.8", "0");
+
+ call("PutItem", """
+ {
+ "TableName": "%s",
+ "Item": {"pk": {"S": "novec"}, "label": {"S": "no vector"}}
+ }
+ """.formatted(DOCS))
+ .statusCode(200);
+ }
+
+ private static void putVector(String key, String x, String y, String z) {
+ call("PutItem", """
+ {
+ "TableName": "%s",
+ "Item": {
+ "pk": {"S": "%s"},
+ "label": {"S": "item %s"},
+ "embedding": {"L": [{"N": "%s"}, {"N": "%s"}, {"N": "%s"}]}
+ }
+ }
+ """.formatted(DOCS, key, key, x, y, z))
+ .statusCode(200);
+ }
+
+ @Test
+ @Order(3)
+ void describeTableRoundTripsTheVectorIndex() {
+ call("DescribeTable", """
+ {"TableName": "%s"}
+ """.formatted(SCHEMA))
+ .statusCode(200)
+ .body("Table.VectorIndexes.size()", equalTo(1))
+ .body("Table.VectorIndexes[0].IndexName", equalTo("schema"))
+ .body("Table.VectorIndexes[0].Dimensions", equalTo(3))
+ .body("Table.VectorIndexes[0].DistanceFunction", equalTo("COSINE"))
+ .body("Table.VectorIndexes[0].VectorAttribute.AttributeName", equalTo("embedding"))
+ .body("Table.VectorIndexes[0].Projection.ProjectionType", equalTo("ALL"))
+ .body("Table.VectorIndexes[0].Projection.NonKeyAttributes", nullValue())
+ .body("Table.VectorIndexes[0].SearchSchema.AttributeName",
+ contains("tenant", "category"))
+ .body("Table.VectorIndexes[0].SearchSchema.SearchSchemaElementType",
+ contains("HASH", "INLINE_FILTER"))
+ .body("Table.VectorIndexes[0].IndexStatus", equalTo("ACTIVE"))
+ .body("Table.VectorIndexes[0].IndexArn",
+ equalTo("arn:aws:dynamodb:us-east-1:000000000000:table/"
+ + SCHEMA + "/index/schema"))
+ .body("Table.VectorIndexes[0].ItemCount", equalTo(0))
+ .body("Table.VectorIndexes[0].IndexSizeBytes", equalTo(0))
+ .body("Table.VectorIndexes[0].Backfilling", nullValue());
+ }
+
+ @Test
+ @Order(4)
+ void searchRanksByCosineDistance() throws Exception {
+ JsonNode response = json("SearchVectors", search(DOCS, "cosine", 4, ""));
+
+ assertEquals(List.of("a", "d", "b", "c"), keysOf(response));
+ assertEquals(List.of(0.0, 0.3999999761581421, 1.0, 2.0), scoresOf(response));
+ }
+
+ @Test
+ @Order(5)
+ void searchRanksByEuclideanDistance() throws Exception {
+ JsonNode response = json("SearchVectors", search(DOCS, "euclid", 4, ""));
+
+ assertEquals(List.of("a", "d", "b", "c"), keysOf(response));
+ assertEquals(List.of(0.0, 0.8944271802902222, 1.4142135381698608, 2.0),
+ scoresOf(response));
+ }
+
+ @Test
+ @Order(6)
+ void searchRanksByDotProductDescending() throws Exception {
+ JsonNode response = json("SearchVectors", search(DOCS, "dotp", 4, ""));
+
+ assertEquals(List.of("a", "d", "b", "c"), keysOf(response));
+ assertEquals(List.of(1.0, 0.6000000238418579, 0.0, -1.0), scoresOf(response));
+ }
+
+ @Test
+ @Order(7)
+ void topKCutsTheResultSet() throws Exception {
+ JsonNode response = json("SearchVectors", search(DOCS, "cosine", 2, ""));
+
+ assertEquals(List.of("a", "d"), keysOf(response));
+ }
+
+ @Test
+ @Order(8)
+ void itemWithoutTheVectorAttributeStaysOutOfTheIndex() throws Exception {
+ JsonNode response = json("SearchVectors", search(DOCS, "cosine", 100, ""));
+
+ assertEquals(4, response.path("SearchResults").size());
+ assertFalse(keysOf(response).contains("novec"));
+ }
+
+ @Test
+ @Order(9)
+ void vectorAttributeIsAbsentUnlessProjected() throws Exception {
+ JsonNode response = json("SearchVectors", search(DOCS, "cosine", 1, ""));
+ JsonNode item = response.path("SearchResults").path(0).path("Item");
+
+ assertEquals("a", item.path("pk").path("S").asText());
+ assertEquals("item a", item.path("label").path("S").asText());
+ assertTrue(item.path("embedding").isMissingNode(), "the vector is not projected by default");
+ }
+
+ @Test
+ @Order(10)
+ void projectionExpressionReturnsTheIndexF32Copy() throws Exception {
+ JsonNode response = json("SearchVectors", search(DOCS, "cosine", 2, """
+ , "ProjectionExpression": "pk, embedding"
+ """));
+ JsonNode results = response.path("SearchResults");
+
+ assertTrue(results.path(0).path("Item").path("label").isMissingNode());
+ assertEquals(List.of("1.0", "0.0", "0.0"), vectorOf(results.path(0)));
+ // The third component was written as "0" and comes back as "0.0": a hit carries the
+ // index's own f32 copy rather than the text the item stores.
+ assertEquals(List.of("0.6", "0.8", "0.0"), vectorOf(results.path(1)));
+ }
+
+ @Test
+ @Order(11)
+ void searchReportsRequestBytesUnderTotalAndIndexesOnly() throws Exception {
+ JsonNode total = json("SearchVectors", search(DOCS, "cosine", 1, """
+ , "ReturnConsumedCapacity": "TOTAL"
+ """));
+ assertEquals(1024.0,
+ total.path("ConsumedCapacity").path("VectorSearchRequestBytes").asDouble());
+ assertTrue(total.path("ConsumedCapacity").path("CapacityUnits").isMissingNode());
+ assertTrue(total.path("ConsumedCapacity").path("TableName").isMissingNode());
+
+ JsonNode indexes = json("SearchVectors", search(DOCS, "cosine", 1, """
+ , "ReturnConsumedCapacity": "INDEXES"
+ """));
+ assertEquals(1024.0,
+ indexes.path("ConsumedCapacity").path("VectorSearchRequestBytes").asDouble());
+
+ JsonNode none = json("SearchVectors", search(DOCS, "cosine", 1, """
+ , "ReturnConsumedCapacity": "NONE"
+ """));
+ assertTrue(none.path("ConsumedCapacity").isMissingNode());
+ }
+
+ @Test
+ @Order(12)
+ void createTableRejectsAVectorIndexOnAProvisionedTable() {
+ expectValidation("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "VectorIndexes": [
+ {"IndexName": "vix",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}
+ ]
+ }
+ """.formatted(REJECTED),
+ "One or more parameter values were invalid: Vector indexes are only supported "
+ + "for PAY_PER_REQUEST tables");
+ }
+
+ @Test
+ @Order(13)
+ void createTableRejectsASearchSchemaAttributeWithNoDefinition() {
+ expectValidation("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "vix",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "SearchSchema": [{"AttributeName": "tenant", "SearchSchemaElementType": "HASH"}],
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}
+ ]
+ }
+ """.formatted(REJECTED),
+ "One or more parameter values were invalid: One element in SearchSchema is not "
+ + "defined in attribute definitions");
+ }
+
+ @Test
+ @Order(14)
+ void createTableRejectsDimensionsAboveTheLimit() {
+ expectValidation("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "vix",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 5000,
+ "DistanceFunction": "COSINE"}
+ ]
+ }
+ """.formatted(REJECTED),
+ "One or more parameter values were invalid: Number of dimensions must be between "
+ + "1 and 4096 inclusive.");
+ }
+
+ @Test
+ @Order(15)
+ void createTableRejectsMoreThanFiveVectorIndexes() {
+ List<String> indexes = new ArrayList<>();
+ for (int i = 1; i <= 6; i++) {
+ indexes.add("""
+ {"IndexName": "vix%d",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}""".formatted(i));
+ }
+ expectValidation("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [%s]
+ }
+ """.formatted(REJECTED, String.join(",\n", indexes)),
+ "One or more parameter values were invalid: VectorIndex count exceeds the "
+ + "per-table limit of 5");
+ }
+
+ @Test
+ @Order(16)
+ void createTableRejectsTwoDimensionsForOneVectorAttribute() {
+ expectValidation("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "three",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"},
+ {"IndexName": "four",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 4,
+ "DistanceFunction": "COSINE"}
+ ]
+ }
+ """.formatted(REJECTED),
+ "One or more parameter values were invalid: Conflicting attribute definition for "
+ + "'embedding'. All VectorIndexes on the same vector attribute must use the same "
+ + "dimensions.");
+ }
+
+ @Test
+ @Order(17)
+ void searchVectorsRejectsATopKAboveTheLimit() {
+ expectValidation("SearchVectors", search(DOCS, "cosine", 101, ""),
+ "Provided TopK value '101' is out of valid range. The value must be between 1 "
+ + "and 100 inclusive");
+ }
+
+ @Test
+ @Order(18)
+ void searchVectorsRequiresAConditionOnTheSearchSchemaHash() {
+ expectValidation("SearchVectors", search(SCHEMA, "schema", 4, ""),
+ "SearchConditionExpression must be provided when SearchSchema has a HASH key");
+ }
+
+ @Test
+ @Order(19)
+ void searchVectorsRejectsANonEqualityComparator() {
+ expectValidation("SearchVectors", search(SCHEMA, "schema", 4, """
+ ,
+ "SearchConditionExpression": "tenant < :t",
+ "ExpressionAttributeValues": {":t": {"S": "acme"}}
+ """),
+ "Invalid SearchConditionExpression: Invalid comparator used in "
+ + "SearchConditionExpression");
+ }
+
+ @Test
+ @Order(20)
+ void searchVectorsRejectsADimensionMismatch() {
+ expectValidation("SearchVectors", """
+ {
+ "TableName": "%s",
+ "IndexName": "plain",
+ "SearchVector": [{"N": "1"}, {"N": "0"}],
+ "TopK": 4
+ }
+ """.formatted(PLAIN),
+ "Input search vector dimension 2 does not match vector index dimension 3");
+ }
+
+ @Test
+ @Order(21)
+ void searchVectorsRejectsAConditionOutsideTheSearchSchema() {
+ expectValidation("SearchVectors", search(PLAIN, "plain", 4, """
+ ,
+ "SearchConditionExpression": "tenant = :t",
+ "ExpressionAttributeValues": {":t": {"S": "acme"}}
+ """),
+ "SearchConditionExpression must not contain any attributes that is not in "
+ + "SearchSchema. Invalid attribute: tenant");
+ }
+
+ @Test
+ @Order(22)
+ void searchVectorsRejectsANonNumericSearchVector() {
+ expectValidation("SearchVectors", """
+ {
+ "TableName": "%s",
+ "IndexName": "plain",
+ "SearchVector": [{"N": "1"}, {"S": "x"}, {"N": "0"}],
+ "TopK": 4
+ }
+ """.formatted(PLAIN),
+ "Search vector contains invalid values. All values in the search vector must be "
+ + "a 32-bit floating-point number attribute");
+ }
+
+ @Test
+ @Order(23)
+ void putItemRejectsAVectorOfTheWrongLength() {
+ expectValidation("PutItem", """
+ {
+ "TableName": "%s",
+ "Item": {
+ "pk": {"S": "short"},
+ "embedding": {"L": [{"N": "1"}, {"N": "0"}]}
+ }
+ }
+ """.formatted(PLAIN),
+ "One or more parameter values were invalid. Invalid size for parameter embedding, "
+ + "Expected: 3, Actual: 2 IndexName: plain");
+ }
+
+ @Test
+ @Order(24)
+ void putItemRejectsANonNumericVectorComponent() {
+ expectValidation("PutItem", """
+ {
+ "TableName": "%s",
+ "Item": {
+ "pk": {"S": "typed"},
+ "embedding": {"L": [{"N": "1"}, {"S": "x"}, {"N": "0"}]}
+ }
+ }
+ """.formatted(PLAIN),
+ "One or more parameter values were invalid. Invalid type for parameter "
+ + "embedding[1], Expected: 32-bit floating point number, Actual: S. "
+ + "IndexName: plain");
+ }
+
+ @Test
+ @Order(25)
+ void putItemRejectsAVectorAttributeThatIsNotAList() {
+ expectValidation("PutItem", """
+ {
+ "TableName": "%s",
+ "Item": {
+ "pk": {"S": "scalar"},
+ "embedding": {"S": "not a vector"}
+ }
+ }
+ """.formatted(PLAIN),
+ "One or more parameter values were invalid. Invalid type for parameter embedding, "
+ + "Expected: 32-bit floating point number list IndexName: plain");
+ }
+
+ @Test
+ @Order(26)
+ void putItemRejectsAnEmptySearchSchemaHashValue() {
+ expectValidation("PutItem", """
+ {
+ "TableName": "%s",
+ "Item": {
+ "pk": {"S": "blank-tenant"},
+ "tenant": {"S": ""},
+ "embedding": {"L": [{"N": "1"}, {"N": "0"}, {"N": "0"}]}
+ }
+ }
+ """.formatted(SCHEMA),
+ "One or more parameter values are not valid. A value specified for a secondary "
+ + "index key is not supported. The AttributeValue for a key attribute cannot "
+ + "contain an empty string value. IndexName: schema, IndexKey: tenant");
+ }
+
+ /**
+ * The conformance suite probes SearchVectors against a table that does not exist to decide
+ * whether the operation is served at all. Any other answer turns all 52 vector tests into
+ * skips rather than failures.
+ */
+ @Test
+ @Order(27)
+ void searchVectorsOnAMissingTableIsResourceNotFound() {
+ call("SearchVectors", search("VectorNoSuchTable", "cosine", 1, ""))
+ .statusCode(400)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ }
+
+ @Test
+ @Order(28)
+ void partiQLRejectsAVectorIndex() {
+ expectValidation("ExecuteStatement", """
+ {"Statement": "SELECT * FROM \\"%s\\".\\"plain\\" WHERE pk = 'a'"}
+ """.formatted(PLAIN),
+ "Scan operation not supported on this index type");
+ }
+
+ @Test
+ @Order(29)
+ void createTableRejectsADistanceFunctionOutsideTheEnum() {
+ for (String distanceFunction : List.of("MANHATTAN", "cosine")) {
+ expectValidation("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "vix",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "%s"}
+ ]
+ }
+ """.formatted(REJECTED, distanceFunction),
+ "1 validation error detected: Value '" + distanceFunction + "' at "
+ + "'vectorIndexes.1.member.distanceFunction' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: [DOT_PRODUCT, COSINE, EUCLIDEAN]");
+ }
+ }
+
+ @Test
+ @Order(30)
+ void updateTableRejectsADistanceFunctionOutsideTheEnum() {
+ expectValidation("UpdateTable", """
+ {
+ "TableName": "%s",
+ "VectorIndexUpdates": [
+ {"Create": {"IndexName": "vix",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "MANHATTAN"}}
+ ]
+ }
+ """.formatted(PLAIN),
+ "1 validation error detected: Value 'MANHATTAN' at "
+ + "'vectorIndexUpdates.1.member.create.distanceFunction' failed to satisfy "
+ + "constraint: Member must satisfy enum value set: [DOT_PRODUCT, COSINE, EUCLIDEAN]");
+ }
+
+ @Test
+ @Order(31)
+ void createTableRequiresDimensionsVectorAttributeAndProjection() {
+ expectValidation("CreateTable", vectorIndexTable("""
+ {"IndexName": "vix",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "DistanceFunction": "COSINE"}"""),
+ requiredMember("dimensions"));
+
+ expectValidation("CreateTable", vectorIndexTable("""
+ {"IndexName": "vix",
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}"""),
+ requiredMember("vectorAttribute"));
+
+ expectValidation("CreateTable", vectorIndexTable("""
+ {"IndexName": "vix",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}"""),
+ requiredMember("projection"));
+ }
+
+ /** A CreateTable body carrying the one vector index given. */
+ private static String vectorIndexTable(String vectorIndex) {
+ return """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [%s]
+ }
+ """.formatted(REJECTED, vectorIndex);
+ }
+
+ private static String requiredMember(String member) {
+ return "1 validation error detected: Value null at 'vectorIndexes.1.member." + member
+ + "' failed to satisfy constraint: Member must not be null";
+ }
+
+ @Test
+ @Order(32)
+ void createTableRejectsAVectorIndexNamedLikeAnotherIndex() {
+ expectValidation("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [
+ {"AttributeName": "pk", "AttributeType": "S"},
+ {"AttributeName": "gsk", "AttributeType": "S"}
+ ],
+ "BillingMode": "PAY_PER_REQUEST",
+ "GlobalSecondaryIndexes": [
+ {"IndexName": "dup",
+ "KeySchema": [{"AttributeName": "gsk", "KeyType": "HASH"}],
+ "Projection": {"ProjectionType": "ALL"}}
+ ],
+ "VectorIndexes": [
+ {"IndexName": "dup",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}
+ ]
+ }
+ """.formatted(REJECTED),
+ "One or more parameter values were invalid: Duplicate index name: dup");
+
+ expectValidation("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "dup",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"},
+ {"IndexName": "dup",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}
+ ]
+ }
+ """.formatted(REJECTED),
+ "One or more parameter values were invalid: Duplicate index name: dup");
+ }
+
+ @Test
+ @Order(33)
+ void searchVectorsChecksTheTopKRangeBeforeTheTableAndTheIndex() {
+ String outOfRange = "Provided TopK value '1000' is out of valid range. The value must be "
+ + "between 1 and 100 inclusive";
+
+ expectValidation("SearchVectors", search("VectorNoSuchTable", "cosine", 1000, ""), outOfRange);
+ expectValidation("SearchVectors", search(DOCS, "VectorNoSuchIndex", 1000, ""), outOfRange);
+ }
+
+ /**
+ * AWS deserialises SearchVectors with serde and points at the body's closing brace, so the
+ * column is the length of the compact JSON an SDK sends. Both bodies below are compact, which
+ * is what pins the rule rather than the shape of the message.
+ */
+ @Test
+ @Order(34)
+ void searchVectorsReportsAnOmittedMemberAtTheBodyLength() {
+ String withoutVector = """
+ {"TableName":"%s","IndexName":"plain","TopK":1}""".formatted(PLAIN);
+ expectValidation("SearchVectors", withoutVector,
+ "missing field `SearchVector` at line 1 column " + withoutVector.length());
+
+ String withoutIndex = """
+ {"TableName":"%s","SearchVector":[{"N":"1"},{"N":"0"},{"N":"0"}],"TopK":1}"""
+ .formatted(PLAIN);
+ expectValidation("SearchVectors", withoutIndex,
+ "missing field `IndexName` at line 1 column " + withoutIndex.length());
+ }
+
+ @Test
+ @Order(35)
+ void updateTableRejectsLeavingPayPerRequestWhileAVectorIndexExists() {
+ expectValidation("UpdateTable", """
+ {"TableName": "%s", "BillingMode": "PROVISIONED"}
+ """.formatted(PLAIN),
+ "One or more parameter values were invalid: Vector indexes are only supported "
+ + "for PAY_PER_REQUEST tables");
+ }
+
+ @Test
+ @Order(36)
+ void updateTableReportsThePositionInTheVectorIndexUpdatesArray() {
+ expectValidation("UpdateTable", """
+ {
+ "TableName": "%s",
+ "VectorIndexUpdates": [
+ {"Delete": {"IndexName": "plain"}},
+ {"Create": {"IndexName": "ab",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}}
+ ]
+ }
+ """.formatted(PLAIN),
+ "1 validation error detected: Value 'ab' at "
+ + "'vectorIndexUpdates.2.member.create.indexName' failed to satisfy constraint: "
+ + "Member must have length greater than or equal to 3");
+
+ call("DescribeTable", """
+ {"TableName": "%s"}
+ """.formatted(PLAIN))
+ .statusCode(200)
+ .body("Table.VectorIndexes.IndexName", contains("plain"));
+ }
+
+ @Test
+ @Order(37)
+ void createTableRejectsAnIncludeProjectionWithNoNonKeyAttributes() {
+ expectValidation("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "vix",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "INCLUDE"},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}
+ ]
+ }
+ """.formatted(REJECTED),
+ "One or more parameter values were invalid: ProjectionType is INCLUDE, but "
+ + "NonKeyAttributes is not specified");
+ }
+
+ @Test
+ @Order(38)
+ void updateTableRejectsAKeysOnlyProjectionWithNonKeyAttributes() {
+ expectValidation("UpdateTable", """
+ {
+ "TableName": "%s",
+ "VectorIndexUpdates": [
+ {"Create": {"IndexName": "vix",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "KEYS_ONLY",
+ "NonKeyAttributes": ["label"]},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}}
+ ]
+ }
+ """.formatted(PLAIN),
+ "One or more parameter values were invalid: ProjectionType is KEYS_ONLY, but "
+ + "NonKeyAttributes is specified");
+ }
+
+ @Test
+ @Order(39)
+ void createTableReportsTheNestedVectorAttributeMember() {
+ expectValidation("CreateTable", """
+ {
+ "TableName": "vec_nested_attr",
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "vix", "VectorAttribute": {},
+ "Projection": {"ProjectionType": "ALL"},
+ "Dimensions": 3, "DistanceFunction": "COSINE"}
+ ]
+ }
+ """,
+ "1 validation error detected: Value null at "
+ + "'vectorIndexes.1.member.vectorAttribute.attributeName' failed to satisfy "
+ + "constraint: Member must not be null");
+ }
+
+ @Test
+ @Order(40)
+ void createTableRequiresADistanceFunction() {
+ expectValidation("CreateTable", """
+ {
+ "TableName": "vec_no_distance",
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "vix", "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "ALL"}, "Dimensions": 3}
+ ]
+ }
+ """,
+ "1 validation error detected: Value null at "
+ + "'vectorIndexes.1.member.distanceFunction' failed to satisfy constraint: "
+ + "Member must not be null");
+ }
+
+ @Test
+ @Order(41)
+ void searchVectorsReportsAnOmittedTopKAtTheBodyLength() {
+ String withoutTopK = """
+ {"TableName":"%s","IndexName":"plain","SearchVector":[{"N":"1"},{"N":"0"},{"N":"0"}]}"""
+ .formatted(PLAIN);
+ expectValidation("SearchVectors", withoutTopK,
+ "missing field `TopK` at line 1 column " + withoutTopK.length());
+ }
+
+ /** SearchVectors is served by its own frontend, which omits the envelope other operations add. */
+ @Test
+ @Order(42)
+ void searchVectorsRejectsAReturnConsumedCapacityWithNoEnvelope() {
+ expectValidation("SearchVectors", """
+ {"TableName": "%s", "IndexName": "plain",
+ "SearchVector": [{"N": "1"}, {"N": "0"}, {"N": "0"}],
+ "TopK": 1, "ReturnConsumedCapacity": "BOGUS"}
+ """.formatted(PLAIN),
+ "Value 'BOGUS' at 'returnConsumedCapacity' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: [INDEXES, TOTAL, NONE]");
+ }
+
+ /** An empty list is a length constraint on the member, the way a GSI reports it. */
+ @Test
+ @Order(43)
+ void createTableRejectsAnEmptyNonKeyAttributesList() {
+ expectValidation("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "VectorIndexes": [
+ {"IndexName": "vix",
+ "VectorAttribute": {"AttributeName": "embedding"},
+ "Projection": {"ProjectionType": "INCLUDE", "NonKeyAttributes": []},
+ "Dimensions": 3,
+ "DistanceFunction": "COSINE"}
+ ]
+ }
+ """.formatted(REJECTED),
+ "1 validation error detected: Value '[]' at "
+ + "'vectorIndexes.1.member.projection.nonKeyAttributes' failed to satisfy "
+ + "constraint: Member must have length greater than or equal to 1");
+ }
+
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java
index 2cd50737a5..1afd676c03 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbWriteCapacityTest.java
@@ -5,7 +5,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;
+import io.github.hectorvent.floci.services.dynamodb.model.SearchSchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -186,4 +188,86 @@ class DynamoDbWriteCapacityTest {
var cost = DynamoDbWriteCapacity.forWrite(table, big, fullItem());
assertEquals(3.0, cost.table(), "the larger of the two images rounds up per 1KB");
}
+
+ /**
+ * Two vector indexes on one attribute, differing only in projection. That pair is what
+ * separates the projection term of VectorWriteRequestBytes from every other term, and it
+ * mirrors the fixture the conformance suite measured against real DynamoDB.
+ */
+ private static TableDefinition vectorTable() {
+ TableDefinition vectorIndexed = new TableDefinition();
+ vectorIndexed.setKeySchema(List.of(new KeySchemaElement("pk", "HASH")));
+ vectorIndexed.setVectorIndexes(List.of(
+ new VectorIndex("vix", "embedding", List.of(), "ALL", List.of(), 3L, "COSINE"),
+ new VectorIndex("vkeys", "embedding", List.of(), "KEYS_ONLY", List.of(), 3L, "COSINE")));
+ return vectorIndexed;
+ }
+
+ private static ObjectNode vectorItem(String key) {
+ return item("""
+ {"pk": {"S": "%s"},
+ "embedding": {"L": [{"N": "1"}, {"N": "0"}, {"N": "0"}]}}
+ """.formatted(key));
+ }
+
+ @Test
+ void vectorEntryBelowTheFloorReportsTheFloor() {
+ DynamoDbWriteCapacity.Cost cost =
+ DynamoDbWriteCapacity.forWrite(vectorTable(), null, vectorItem("f0"));
+ assertEquals(Map.of("vix", 1024.0, "vkeys", 1024.0), cost.vectorBytes());
+ assertEquals(1.0, cost.total(), "bytes processed are not capacity units");
+ }
+
+ @Test
+ void vectorFloorBindsPerEntryRatherThanPerWrite() {
+ ObjectNode blobbed = vectorItem("3a");
+ blobbed.set("blob", item("""
+ {"S": "%s"}
+ """.formatted("x".repeat(1500))));
+ DynamoDbWriteCapacity.Cost cost = DynamoDbWriteCapacity.forWrite(vectorTable(), null, blobbed);
+ // 4 for the key, 1504 for the blob, 9 for the vector's name, 4 per dimension.
+ assertEquals(1529.0, cost.vectorBytes().get("vix"));
+ assertEquals(1024.0, cost.vectorBytes().get("vkeys"), "KEYS_ONLY never holds the blob");
+ }
+
+ @Test
+ void identicalOverwriteChargesNoVectorWrite() {
+ DynamoDbWriteCapacity.Cost cost =
+ DynamoDbWriteCapacity.forWrite(vectorTable(), vectorItem("a"), vectorItem("a"));
+ assertTrue(cost.vectorBytes().isEmpty(), "replication is delta-based");
+ }
+
+ @Test
+ void vectorIndexIsChargedOnlyWhenItProjectsTheChange() {
+ ObjectNode before = vectorItem("a");
+ before.set("note", item("""
+ {"S": "before"}
+ """));
+ ObjectNode after = vectorItem("a");
+ after.set("note", item("""
+ {"S": "after!"}
+ """));
+ DynamoDbWriteCapacity.Cost cost = DynamoDbWriteCapacity.forWrite(vectorTable(), before, after);
+ assertEquals(1024.0, cost.vectorBytes().get("vix"));
+ assertNull(cost.vectorBytes().get("vkeys"), "the KEYS_ONLY entry did not change");
+ }
+
+ @Test
+ void itemWithoutTheVectorAttributeEntersNoVectorIndex() {
+ DynamoDbWriteCapacity.Cost cost = DynamoDbWriteCapacity.forWrite(vectorTable(), null, item("""
+ {"pk": {"S": "plain"}, "label": {"S": "no-vector"}}
+ """));
+ assertTrue(cost.vectorBytes().isEmpty());
+ }
+
+ @Test
+ void itemMissingTheSearchSchemaHashEntersNoVectorIndex() {
+ TableDefinition partitioned = new TableDefinition();
+ partitioned.setKeySchema(List.of(new KeySchemaElement("pk", "HASH")));
+ partitioned.setVectorIndexes(List.of(new VectorIndex("schema", "embedding",
+ List.of(new SearchSchemaElement("tenant", "HASH")), "ALL", List.of(), 3L, "COSINE")));
+ DynamoDbWriteCapacity.Cost cost =
+ DynamoDbWriteCapacity.forWrite(partitioned, null, vectorItem("no-tenant"));
+ assertTrue(cost.vectorBytes().isEmpty());
+ }
}
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 5909b86912..61ba4dd749 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -137,6 +137,8 @@ floci:
global-bucket-namespace: false
dynamodb:
enabled: true
+ vector-index-allocation-seconds: 0
+ vector-index-backfill-seconds: 0
sns:
enabled: true
lambda: