Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(storage): stabilize vector scroll pagination (#4932)

zhanggaoyuane797708e7876
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.py
index 9de47266a..06555b939 100644
--- a/openviking/storage/vectordb/collection/local_collection.py
+++ b/openviking/storage/vectordb/collection/local_collection.py
@@ -753,9 +753,9 @@ class LocalCollection(ICollection):
new_filters["filter"] = filters
# Copy output_fields to avoid modifying the original list
- if output_fields is None:
- output_fields_copy = [field]
- remove_field = True
+ if not output_fields:
+ output_fields_copy = None
+ remove_field = False
else:
output_fields_copy = list(output_fields)
if field not in output_fields_copy:
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index 61b9931f0..c2bf91055 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -606,29 +606,6 @@ class _SingleAccountBackend:
order_desc=order_desc,
)
- async def strict_scroll(
- self,
- filter: Optional[Dict[str, Any] | FilterExpr] = None,
- limit: int = 100,
- cursor: Optional[str] = None,
- output_fields: Optional[List[str]] = None,
- ) -> tuple[List[Dict[str, Any]], Optional[str]]:
- """Return a stable URI-ordered page for a transactional scan."""
- offset = int(cursor) if cursor else 0
- records = await self.strict_query(
- filter=filter,
- limit=limit,
- offset=offset,
- output_fields=output_fields,
- # The local engine's scalar sorter does not return records for
- # path/string fields. ``updated_at`` is an indexed date-time field
- # on every context collection and provides stable offset pages.
- order_by="updated_at",
- order_desc=False,
- )
- next_cursor = str(offset + len(records)) if len(records) == limit else None
- return records, next_cursor
-
async def strict_count(self, filter: Optional[Dict[str, Any] | FilterExpr] = None) -> int:
"""Count transaction records without converting backend errors to zero."""
return int(
@@ -823,22 +800,20 @@ class _SingleAccountBackend:
cursor: Optional[str] = None,
output_fields: Optional[List[str]] = None,
) -> tuple[List[Dict[str, Any]], Optional[str]]:
- """Scroll records without converting backend failures into an empty page."""
- if isinstance(filter, dict):
- filter = RawDSL(filter)
- if self._bound_account_id:
- account_filter = Eq("account_id", self._bound_account_id)
- filter = And([account_filter, filter]) if filter else account_filter
-
+ """Return an updated_at-ordered page and propagate backend failures."""
offset = int(cursor) if cursor else 0
- records = await self._async_adapter.call(
- "query",
+ records = await self.strict_query(
filter=filter,
limit=limit,
offset=offset,
output_fields=output_fields,
+ # The local engine's scalar sorter does not return records for
+ # path/string fields. ``updated_at`` is an indexed date-time field
+ # on every context collection and provides stable offset pages.
+ order_by="updated_at",
+ order_desc=False,
)
- next_cursor = str(offset + limit) if len(records) == limit else None
+ next_cursor = str(offset + len(records)) if len(records) == limit else None
return records, next_cursor
async def count(self, filter: Optional[Dict[str, Any] | FilterExpr] = None) -> int:
@@ -1448,7 +1423,7 @@ class VikingVectorIndexBackend:
output_fields: List[str],
) -> tuple[List[Dict[str, Any]], Optional[str]]:
backend = self._get_backend_for_context(ctx)
- return await backend.strict_scroll(
+ return await backend.scroll(
filter=filter,
limit=limit,
cursor=cursor,
diff --git a/tests/storage/test_vector_transfer.py b/tests/storage/test_vector_transfer.py
index 791a78747..2ba6654bf 100644
--- a/tests/storage/test_vector_transfer.py
+++ b/tests/storage/test_vector_transfer.py
@@ -307,6 +307,12 @@ async def test_copy_uri_mapping_scans_real_local_path_records(tmp_path):
assert result.scanned == 1
copied = await backend.get_context_by_uri(target, ctx=_ctx())
assert [record["uri"] for record in copied] == [target]
+
+ first_page, cursor = await backend.scroll(limit=1, ctx=_ctx())
+ assert cursor is not None
+ last_page, cursor = await backend.scroll(limit=2, cursor=cursor, ctx=_ctx())
+ assert cursor is None
+ assert [record["uri"] for record in first_page + last_page] == [source, target]
finally:
await backend.close()
@@ -370,7 +376,7 @@ async def test_get_l2_abstracts_by_uris_uses_strict_batched_lookup():
@pytest.mark.asyncio
-async def test_strict_scroll_propagates_real_adapter_query_failure():
+async def test_scroll_propagates_real_adapter_query_failure():
backend = _SingleAccountBackend.__new__(_SingleAccountBackend)
backend._bound_account_id = "acct"
backend._async_adapter = SimpleNamespace(
@@ -378,7 +384,7 @@ async def test_strict_scroll_propagates_real_adapter_query_failure():
)
with pytest.raises(RuntimeError, match="injected query failure"):
- await backend.strict_scroll(limit=100, output_fields=["id", "uri"])
+ await backend.scroll(limit=100, output_fields=["id", "uri"])
@pytest.mark.asyncio