Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(storage): hydrate incremental records by primary key (#5258)
Jiahui Zhou511b13f39456
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex ce1dc4c8d..69ef75e57 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -2003,7 +2003,13 @@ class VikingVectorIndexBackend:batch_size: int = 100,output_fields: Optional[Container[str]] = None,) -> Dict[str, Dict[str, Any]]:- """Load requested non-vector fields, using strict DSL before ID fallback."""+ """Load requested non-vector fields by primary-key point-get.++ ``expected`` maps each known record id to its ``{uri, level}`` identity.+ Records are fetched by primary key (never filtered on ``id``), and the+ selected non-vector fields are projected on the client side, so the+ returned rows never carry ``vector``/``sparse_vector``/``content``.+ """if batch_size <= 0:raise ValueError("batch_size must be positive")requested_ids = list(expected)@@ -2060,29 +2066,15 @@ class VikingVectorIndexBackend:for start in range(0, len(requested_ids), batch_size):chunk = requested_ids[start : start + batch_size]- cursor: Optional[str] = None- seen_cursors: set[str] = set()- while True:- page, next_cursor = await self._strict_transfer_page(- ctx,- In("id", chunk),- limit=batch_size,- cursor=cursor,- output_fields=selected_fields,- )- for record in page:- _accept(record)- if next_cursor is None:- break- if next_cursor in seen_cursors:- raise RuntimeError(f"Incremental hydration cursor repeated: {next_cursor}")- seen_cursors.add(next_cursor)- cursor = next_cursor-- missing = [record_id for record_id in chunk if record_id not in hydrated]- if missing:- for record in await self._strict_transfer_get(ctx, missing):- _accept(record)+ # ``id`` is the collection primary key. VikingDB only accepts ``must``+ # filters on ScalarIndex fields, and a primary key is intentionally not+ # indexed as a scalar (it is addressed by point-get). Filtering on it+ # (In("id", ...)) is rejected by strict backends, so fetch the records+ # by primary key directly: no scalar index, no ordering, no offset+ # pagination. Records that no longer exist are simply absent, and the+ # planner promotes those dependencies separately.+ for record in await self._strict_transfer_get(ctx, chunk):+ _accept(record)return hydratedasync def delete_account_data(self, account_id: str, *, ctx: RequestContext) -> int:diff --git a/tests/storage/test_vector_transfer.py b/tests/storage/test_vector_transfer.pyindex d3cbd35a7..fdb2c47b3 100644--- a/tests/storage/test_vector_transfer.py+++ b/tests/storage/test_vector_transfer.py@@ -715,13 +715,15 @@ async def test_incremental_inventory_projects_request_scalar_fields():@pytest.mark.asyncio-async def test_incremental_hydration_uses_dsl_then_fetches_only_missing_ids():+async def test_incremental_hydration_fetches_records_by_primary_key():root = "viking://resources/docs"first = _record("root-l0", root, level=0, abstract="root abstract")second = _record("file-l2", f"{root}/a.py", level=2, md5="ma")backend = _MemoryTransferBackend([first, second])- backend._strict_transfer_page = AsyncMock(return_value=([dict(first)], None))- backend._strict_transfer_get = AsyncMock(return_value=[dict(second)])+ backend._strict_transfer_page = AsyncMock(+ side_effect=AssertionError("hydration must not filter on the primary key")+ )+ backend._strict_transfer_get = AsyncMock(return_value=[dict(first), dict(second)])records = await backend.hydrate_incremental_records({@@ -734,37 +736,34 @@ async def test_incremental_hydration_uses_dsl_then_fetches_only_missing_ids():assert set(records) == {"root-l0", "file-l2"}assert records["root-l0"]["abstract"] == "root abstract"assert records["file-l2"]["md5"] == "ma"- backend._strict_transfer_get.assert_awaited_once_with(_ctx(), ["file-l2"])- query = backend._strict_transfer_page.await_args- assert query.args[1] == In("id", ["root-l0", "file-l2"])- assert "vector" not in query.kwargs["output_fields"]- assert "sparse_vector" not in query.kwargs["output_fields"]+ # Point-get is used, never an In("id", ...) scalar filter.+ backend._strict_transfer_page.assert_not_awaited()+ backend._strict_transfer_get.assert_awaited_once_with(_ctx(), ["root-l0", "file-l2"])+ # Vector payloads returned by the fetch are dropped by client-side projection.+ for hydrated in records.values():+ assert "vector" not in hydrated+ assert "sparse_vector" not in hydrated+ assert "content" not in hydrated@pytest.mark.asyncio-async def test_incremental_hydration_propagates_dsl_failure_without_fallback():+async def test_incremental_hydration_propagates_fetch_failure():backend = _MemoryTransferBackend([])- backend._strict_transfer_page = AsyncMock(side_effect=RuntimeError("query failed"))- backend._strict_transfer_get = AsyncMock()+ backend._strict_transfer_get = AsyncMock(side_effect=RuntimeError("fetch failed"))- with pytest.raises(RuntimeError, match="query failed"):+ with pytest.raises(RuntimeError, match="fetch failed"):await backend.hydrate_incremental_records({"file-l2": {"uri": "viking://resources/docs/a.py", "level": 2}},ctx=_ctx(),)- backend._strict_transfer_get.assert_not_awaited()-@pytest.mark.asyncioasync def test_incremental_hydration_rejects_mismatched_record_identity():root = "viking://resources/docs"backend = _MemoryTransferBackend([])- backend._strict_transfer_page = AsyncMock(- return_value=(- [_record("file-l2", f"{root}/wrong.py", level=2)],- None,- )+ backend._strict_transfer_get = AsyncMock(+ return_value=[_record("file-l2", f"{root}/wrong.py", level=2)])with pytest.raises(RuntimeError, match="identity mismatch"):@@ -777,7 +776,6 @@ async def test_incremental_hydration_rejects_mismatched_record_identity():@pytest.mark.asyncioasync def test_incremental_hydration_returns_only_records_still_present():backend = _MemoryTransferBackend([])- backend._strict_transfer_page = AsyncMock(return_value=([], None))backend._strict_transfer_get = AsyncMock(return_value=[])records = await backend.hydrate_incremental_records(@@ -812,19 +810,17 @@ async def test_incremental_hydration_projects_dynamic_non_vector_schema_fields()]})- backend._strict_transfer_page = AsyncMock(return_value=([record], None))hydrated = await backend.hydrate_incremental_records({"file-l2": {"uri": f"{root}/a.py", "level": 2}},ctx=_ctx(),)+ # Dynamic non-vector schema fields survive; vector payloads are dropped.assert hydrated["file-l2"]["business_priority"] == 7- fields = backend._strict_transfer_page.await_args.kwargs["output_fields"]- assert "business_priority" in fields- assert "vector" not in fields- assert "sparse_vector" not in fields- assert "content" not in fields+ assert "vector" not in hydrated["file-l2"]+ assert "sparse_vector" not in hydrated["file-l2"]+ assert "content" not in hydrated["file-l2"]@pytest.mark.asyncio@@ -839,17 +835,8 @@ async def test_incremental_hydration_honors_explicit_summary_projection():)backend = _MemoryTransferBackend([record])backend.get_collection_meta = AsyncMock(- return_value={- "Fields": [- {"FieldName": "id"},- {"FieldName": "uri"},- {"FieldName": "level"},- {"FieldName": "abstract"},- {"FieldName": "business_priority"},- ]- }+ side_effect=AssertionError("explicit projection must not read collection meta"))- backend._strict_transfer_page = AsyncMock(return_value=([record], None))hydrated = await backend.hydrate_incremental_records({"file-l2": {"uri": f"{root}/a.py", "level": 2}},@@ -857,15 +844,14 @@ async def test_incremental_hydration_honors_explicit_summary_projection():output_fields={"abstract"},)+ # Explicit projection keeps only id/uri/level plus the requested field, even+ # though the point-get returns every stored column.assert hydrated["file-l2"] == {"id": "file-l2","uri": f"{root}/a.py","level": 2,"abstract": "summary",}- fields = backend._strict_transfer_page.await_args.kwargs["output_fields"]- assert fields == ["id", "uri", "level", "abstract"]- backend.get_collection_meta.assert_not_awaited()@pytest.mark.asyncio