Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(embedding): truncate resource abstracts before vector writes (#2779)

Evoe2a29a774879
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index 522825541..ce6ea4880 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -22,6 +22,24 @@ from openviking_cli.utils import VikingURI, get_logger
from openviking_cli.utils.config import get_openviking_config
logger = get_logger(__name__)
+
+# The `abstract` scalar is persisted as a vector-store bytes_row string field,
+# which is length-prefixed with a uint16 (STRING_MAX_UINT16_LENGTH = 65535). An
+# oversized abstract raises "string field 'abstract' exceeds 65535 bytes" and
+# fails embedding enqueue, so the resource is silently never vectorized (and thus
+# not retrievable). Cap it with headroom, mirroring
+# memory_updater._truncate_memory_abstract introduced for the memory path (#2774).
+_ABSTRACT_MAX_BYTES = 50_000
+
+
+def _truncate_abstract_bytes(abstract: str) -> str:
+ """Cap an abstract scalar below the vector-store bytes_row byte limit."""
+ encoded = (abstract or "").encode("utf-8")
+ if len(encoded) <= _ABSTRACT_MAX_BYTES:
+ return abstract or ""
+ return encoded[:_ABSTRACT_MAX_BYTES].decode("utf-8", errors="ignore")
+
+
_PORTABLE_SCALAR_FIELDS = frozenset(
{
"type",
@@ -262,6 +280,12 @@ async def vectorize_directory_meta(
created_at, updated_at = await _resolve_context_timestamps(uri, ctx)
+ # Cap the abstract scalar below the bytes_row 65535-byte limit. #2774
+ # added this for the memory path; the resource indexing paths (here and
+ # index_resource, which feeds this function) were missed, so an
+ # .abstract.md / overview > 65535 UTF-8 bytes still fails embedding enqueue.
+ abstract = _truncate_abstract_bytes(abstract)
+
# Vectorize L0: .abstract.md (abstract)
context_abstract = Context(
uri=uri,
@@ -367,6 +391,8 @@ async def vectorize_file(
file_name = summary_dict.get("name") or os.path.basename(file_path)
summary = summary_dict.get("summary", "")
+ # Cap below the bytes_row 65535-byte abstract-scalar limit (#2774 parity).
+ summary = _truncate_abstract_bytes(summary)
created_at, updated_at = await _resolve_context_timestamps(
file_path,
diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py
index 538937963..a0e4df68c 100644
--- a/tests/unit/test_vectorize_file_strategy.py
+++ b/tests/unit/test_vectorize_file_strategy.py
@@ -137,3 +137,74 @@ async def test_index_resource_skips_session_namespace(monkeypatch):
)
assert queue.items == []
+
+
+def test_truncate_abstract_bytes_caps_below_byte_limit():
+ # small values pass through unchanged
+ assert embedding_utils._truncate_abstract_bytes("small") == "small"
+ assert embedding_utils._truncate_abstract_bytes("") == ""
+ # oversized value is capped AND stays valid UTF-8 (no split multibyte char)
+ big = "你" * 30_000 # 90,000 UTF-8 bytes, over the 65535 bytes_row cap
+ capped = embedding_utils._truncate_abstract_bytes(big)
+ encoded = capped.encode("utf-8")
+ assert len(encoded) <= embedding_utils._ABSTRACT_MAX_BYTES
+ assert encoded.decode("utf-8") == capped
+
+
+async def test_vectorize_file_truncates_oversized_abstract(monkeypatch):
+ """An oversized file summary must be capped before it becomes the `abstract`
+ scalar, otherwise the vector-store bytes_row write fails (string field >
+ 65535 bytes) and the resource is silently never vectorized."""
+ queue = DummyQueue()
+ monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS("ignored"))
+ monkeypatch.setattr(
+ embedding_utils,
+ "get_openviking_config",
+ lambda: types.SimpleNamespace(
+ embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000)
+ ),
+ )
+ monkeypatch.setattr(
+ embedding_utils.EmbeddingMsgConverter, "from_context", lambda context: context
+ )
+
+ oversized = "你" * 30_000 # 90,000 UTF-8 bytes
+ await embedding_utils.vectorize_file(
+ file_path="viking://user/default/resources/big.md",
+ summary_dict={"name": "big.md", "summary": oversized},
+ parent_uri="viking://user/default/resources",
+ ctx=DummyReq(),
+ )
+
+ assert len(queue.items) == 1
+ abstract = queue.items[0].abstract
+ assert len(abstract.encode("utf-8")) <= embedding_utils._ABSTRACT_MAX_BYTES
+ assert abstract.encode("utf-8").decode("utf-8") == abstract # valid UTF-8
+
+
+async def test_vectorize_directory_meta_truncates_oversized_abstract(monkeypatch):
+ """The directory-meta path (fed by index_resource reading .abstract.md) must
+ cap the abstract scalar on every enqueued Context (abstract + overview)."""
+ queue = DummyQueue()
+ monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS("ignored"))
+ monkeypatch.setattr(
+ embedding_utils.EmbeddingMsgConverter, "from_context", lambda context: context
+ )
+
+ oversized = "你" * 30_000 # 90,000 UTF-8 bytes
+ await embedding_utils.vectorize_directory_meta(
+ uri="viking://user/default/resources/dir",
+ abstract=oversized,
+ overview="overview text",
+ ctx=DummyReq(),
+ )
+
+ assert queue.items # at least the abstract-level Context was enqueued
+ for item in queue.items:
+ assert isinstance(item, Context)
+ assert len(item.abstract.encode("utf-8")) <= embedding_utils._ABSTRACT_MAX_BYTES
+ assert item.abstract.encode("utf-8").decode("utf-8") == item.abstract