Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
import asyncio
import base64
import hashlib
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
import openviking.storage.content_write as content_write_module
from openviking.server.identity import RequestContext, Role
from openviking.service.task_store import PersistentTaskStore
from openviking.service.task_tracker import TaskTracker
from openviking.session.memory.dataclass import MemoryFile
from openviking.session.memory.utils import MemoryFileUtils
from openviking.storage.acl import AclSpec, AclUpdate
from openviking.storage.content_write import ContentWriteCoordinator
from openviking.storage.context_update_plan import (
ContentTreeAction,
ContextUpdatePlan,
FileRefreshIntent,
)
from openviking.storage.queuefs.queue_middleware import EnqueueReceipt
from openviking.storage.queuefs.semantic_ops.freshness_policy import FreshnessAction
from openviking.telemetry.context import bind_telemetry
from openviking.telemetry.operation import OperationTelemetry
from openviking.utils.content_hash import content_md5
from openviking_cli.exceptions import (
InvalidArgumentError,
NotFoundError,
)
from openviking_cli.session.user_id import UserIdentifier
from tests.test_task_tracker import _FakeAgfs
class _PathLockClient:
def __init__(self):
self.held = False
self.releases = 0
self.handoffs = 0
self.paths = []
async def pathlock_acquire_exact_batch(self, paths):
self.paths = list(paths)
self.held = True
return {"lease_ref": "lock-1"}
async def pathlock_acquire_exact(self, path):
self.paths = [path]
self.held = True
return {"lease_ref": "lock-1"}
async def pathlock_to_handoff(self, lease):
assert lease == {"lease_ref": "lock-1"}
assert self.held is True
return {
"owner_id": "lock-1",
"covered_paths": [{"path": path, "kind": "exact"} for path in self.paths],
}
async def pathlock_handoff(self, lease):
assert lease == {"lease_ref": "lock-1"}
assert self.held is True
self.held = False
self.handoffs += 1
async def pathlock_adopt(self, handoff):
assert handoff["owner_id"] == "lock-1"
self.held = True
return {"lease_ref": "lock-1"}
async def pathlock_release(self, lease):
assert lease == {"lease_ref": "lock-1"}
self.held = False
self.releases += 1
class _VFS:
def __init__(self, root, files=None, fail_uri=None):
self.root = root
self.files = dict(files or {})
self.fail_uri = fail_uri
self.writes = []
self._async_agfs = _PathLockClient()
async def _ensure_access(self, uri, ctx, *, action):
del uri, ctx, action
def _uri_to_path(self, uri, ctx=None):
del ctx
return "/virtual/" + uri.removeprefix("viking://")
async def stat(self, uri, ctx=None, skip_count=False):
del ctx
assert skip_count is True
if uri == self.root:
return {"uri": uri, "isDir": True}
if uri in self.files:
return {"uri": uri, "isDir": False}
raise NotFoundError(uri, "file")
async def read_file(self, uri, ctx=None):
del ctx
if uri not in self.files:
raise NotFoundError(uri, "file")
return self.files[uri]
async def read_file_bytes(self, uri, ctx=None):
value = await self.read_file(uri, ctx=ctx)
return value.encode() if isinstance(value, str) else value
async def write_file(self, uri, content, ctx=None, lease_ref=None):
del ctx
assert lease_ref is not None
if uri == self.fail_uri:
raise OSError("injected write failure")
self.files[uri] = content
self.writes.append(uri)
async def write_file_bytes(self, uri, content, ctx=None, lease_ref=None):
del ctx
assert lease_ref is not None
if uri == self.fail_uri:
raise OSError("injected write failure")
try:
self.files[uri] = content.decode("utf-8")
except UnicodeDecodeError:
self.files[uri] = content
self.writes.append(uri)
@pytest.mark.asyncio
async def test_system_write_is_file_only_without_queue_work(
_coordinator_queue_for_unit_tests,
):
root = "viking://system/memory/memory_templates/custom/demo"
uri = f"{root}/template.md"
vfs = _VFS(root)
coordinator = ContentWriteCoordinator(vfs)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ADMIN)
result = await coordinator.write(
uri=uri,
content="template",
ctx=ctx,
wait=False,
)
assert vfs.files[uri] == "template"
assert vfs._async_agfs.releases == 1
assert vfs._async_agfs.handoffs == 0
assert result["root_uri"] == root
assert result["semantic_status"] == "skipped"
assert result["vector_status"] == "skipped"
assert result["queue_status"] is None
assert _last_content_write_refresh(_coordinator_queue_for_unit_tests) == {"kind": "none"}
_coordinator_queue_for_unit_tests.content_write.enqueue.assert_awaited_once()
_coordinator_queue_for_unit_tests.semantic.enqueue.assert_not_awaited()
@pytest.mark.asyncio
async def test_system_batch_write_is_file_only_without_queue_work(
_coordinator_queue_for_unit_tests,
):
root = "viking://system/memory/memory_templates/custom/demo"
first = f"{root}/first.md"
second = f"{root}/second.md"
vfs = _VFS(root, {first: "old"})
coordinator = ContentWriteCoordinator(vfs)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ADMIN)
result = await coordinator.batch_write(
root_uri=root,
operations=[
{"uri": first, "content": "new"},
{"uri": second, "content": "created"},
],
ctx=ctx,
wait=False,
)
assert result["updated"] == [first]
assert result["created"] == [second]
assert result["semantic_status"] == "skipped"
assert result["vector_status"] == "skipped"
assert result["queue_status"] is None
assert vfs._async_agfs.releases == 1
assert vfs._async_agfs.handoffs == 0
assert _last_content_write_refresh(_coordinator_queue_for_unit_tests) == {"kind": "none"}
_coordinator_queue_for_unit_tests.content_write.enqueue.assert_awaited_once()
_coordinator_queue_for_unit_tests.semantic.enqueue.assert_not_awaited()
@pytest.mark.asyncio
async def test_system_builtin_batch_write_accepts_binary_assets(
_coordinator_queue_for_unit_tests,
):
root = "viking://system/compile/example-skills"
uri = f"{root}/demo/assets/icon.bin"
content = b"\x89PNG\r\n\x1a\nasset"
vfs = _VFS(root)
coordinator = ContentWriteCoordinator(vfs)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
result = await coordinator.batch_write(
root_uri=root,
operations=[
{
"uri": uri,
"content_base64": base64.b64encode(content).decode("ascii"),
"mode": "replace",
}
],
ctx=ctx,
wait=False,
)
assert result["created"] == [uri]
assert vfs.files[uri] == content
assert result["semantic_status"] == "skipped"
assert result["vector_status"] == "skipped"
assert _last_content_write_refresh(_coordinator_queue_for_unit_tests) == {"kind": "none"}
_coordinator_queue_for_unit_tests.content_write.enqueue.assert_awaited_once()
_coordinator_queue_for_unit_tests.semantic.enqueue.assert_not_awaited()
@pytest.mark.asyncio
async def test_user_skill_namespace_batch_write_remains_rejected():
root = "viking://user/default/skills"
coordinator = ContentWriteCoordinator(_VFS(root))
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
with pytest.raises(
InvalidArgumentError,
match="batch-write root must be a resource or memory directory",
):
await coordinator.batch_write(
root_uri=root,
operations=[
{
"uri": f"{root}/demo/SKILL.md",
"content": "skill",
"mode": "replace",
}
],
ctx=ctx,
)
@pytest.mark.asyncio
async def test_system_write_rejects_indexed_metadata():
root = "viking://system/memory/memory_templates/custom/demo"
uri = f"{root}/template.md"
coordinator = ContentWriteCoordinator(_VFS(root))
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ADMIN)
with pytest.raises(InvalidArgumentError, match="tags are not supported"):
await coordinator.write(
uri=uri,
content="template",
tags=["kind=template"],
ctx=ctx,
)
@pytest.mark.asyncio
async def test_system_set_tags_rejects_capability_before_write_access():
uri = "viking://system/memory/memory_templates/custom/demo/template.md"
ensure_access = AsyncMock(side_effect=AssertionError("write access must not be checked"))
coordinator = ContentWriteCoordinator(SimpleNamespace(_ensure_access=ensure_access))
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
with pytest.raises(InvalidArgumentError, match="tags are not supported"):
await coordinator.set_tags(
uri=uri,
tags=["kind=template"],
ctx=ctx,
)
ensure_access.assert_not_awaited()
@pytest.fixture(autouse=True)
def _coordinator_queue_for_unit_tests(monkeypatch, request):
if "service" in request.fixturenames:
yield None
return
semantic_queue = SimpleNamespace(
enqueue=AsyncMock(return_value=EnqueueReceipt("queued", "semantic-work-1"))
)
content_write_queue = SimpleNamespace(
enqueue=AsyncMock(return_value=EnqueueReceipt("queued", "content-write-work-1"))
)
queues = {
"semantic": semantic_queue,
"ContentWrite": content_write_queue,
}
queue_manager = SimpleNamespace(
SEMANTIC="semantic",
CONTENT_WRITE="ContentWrite",
get_queue=lambda name, *args, **kwargs: queues[name],
)
monkeypatch.setattr(content_write_module, "get_queue_manager", lambda: queue_manager)
tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))
monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)
yield SimpleNamespace(semantic=semantic_queue, content_write=content_write_queue)
def _last_content_write_payload(queue_fixture):
return queue_fixture.content_write.enqueue.await_args.args[0]
def _last_content_write_refresh(queue_fixture):
return _last_content_write_payload(queue_fixture)["refresh"]
@pytest.mark.asyncio
async def test_batch_coalesces_compile_target_updates(service, monkeypatch):
"""A batch saves its managed target's content-update time once."""
ctx = RequestContext(user=service.user, role=Role.USER)
root = "viking://resources/batch-metadata"
store = service.compile.lifecycle.store
await service.viking_fs.write_file(f"{root}/seed.md", "seed", ctx=ctx)
await store.register(ctx.account_id, root)
update_time = AsyncMock(wraps=store.mark_content_updated)
monkeypatch.setattr(store, "mark_content_updated", update_time)
ops = [{"uri": f"{root}/{i}.md", "content": "changed"} for i in range(100)]
await service.fs.batch_write(root_uri=root, operations=ops, ctx=ctx, defer_processing=True)
update_time.assert_awaited_once()
@pytest.mark.asyncio
async def test_deferred_batch_versions_content_before_dispatch_without_refresh(
_coordinator_queue_for_unit_tests,
):
"""Compile publication records bytes under its lease without scheduling index work."""
root = "viking://resources/wiki"
uri = f"{root}/page.md"
vfs = _VFS(root)
async def commit(uris, *, lease_ref, write):
assert uris == [uri] and lease_ref == {"lease_ref": "lock-1"}
assert vfs._async_agfs.held
return await write()
commit_content = AsyncMock(side_effect=commit)
commit_batch = AsyncMock()
coordinator = ContentWriteCoordinator(
vfs, commit_content=commit_content, commit_batch=commit_batch
)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
result = await coordinator.batch_write(
root_uri=root,
operations=[{"uri": uri, "content": "x", "mode": "upsert"}],
ctx=ctx,
wait=False,
defer_processing=True,
)
commit_content.assert_awaited_once()
commit_batch.assert_awaited_once_with(lease_ref={"lease_ref": "lock-1"})
assert vfs.writes == [uri]
assert result["semantic_status"] == result["vector_status"] == "deferred"
assert _last_content_write_refresh(_coordinator_queue_for_unit_tests) == {"kind": "none"}
assert vfs._async_agfs.releases == 1 and vfs._async_agfs.handoffs == 0
@pytest.mark.asyncio
async def test_batch_accepts_more_than_256_operations(monkeypatch):
root = "viking://resources/wiki"
vfs = _VFS(root)
coordinator = ContentWriteCoordinator(vfs)
async def refresh(**kwargs):
del kwargs
return None
monkeypatch.setattr(coordinator, "_refresh_batch", refresh)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
def operations(prefix, count):
return [
{
"uri": f"{root}/{prefix}-{index}.txt",
"content": "x",
"mode": "upsert",
}
for index in range(count)
]
result = await coordinator.batch_write(
root_uri=root,
operations=operations("allowed", 256),
ctx=ctx,
wait=False,
)
assert len(result["created"]) == 256
overflow = await coordinator.batch_write(
root_uri=root,
operations=operations("overflow", 257),
ctx=ctx,
wait=False,
)
assert len(overflow["created"]) == 257
@pytest.mark.asyncio
async def test_batch_create_replace_and_upsert_overwrite_or_create(monkeypatch):
root = "viking://resources/wiki"
existing = f"{root}/existing.md"
created_by_replace = f"{root}/replace.md"
created_by_upsert = f"{root}/upsert.md"
vfs = _VFS(root, {existing: "newer"})
coordinator = ContentWriteCoordinator(vfs)
async def refresh(**kwargs):
del kwargs
monkeypatch.setattr(coordinator, "_refresh_batch", refresh)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
result = await coordinator.batch_write(
root_uri=root,
operations=[
{"uri": existing, "content": "replacement", "mode": "create"},
{"uri": created_by_replace, "content": "created", "mode": "replace"},
{"uri": created_by_upsert, "content": "upsert", "mode": "upsert"},
],
ctx=ctx,
wait=False,
)
assert result["updated"] == [existing]
assert result["created"] == [created_by_replace, created_by_upsert]
assert vfs.files == {
existing: "replacement",
created_by_replace: "created",
created_by_upsert: "upsert",
}
@pytest.mark.asyncio
async def test_batch_missing_append_creates_its_initial_body(monkeypatch):
root = "viking://resources/wiki"
page = f"{root}/page.md"
vfs = _VFS(root)
coordinator = ContentWriteCoordinator(vfs)
async def refresh(**kwargs):
del kwargs
monkeypatch.setattr(coordinator, "_refresh_batch", refresh)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
result = await coordinator.batch_write(
root_uri=root,
operations=[{"uri": page, "content": "initial body", "mode": "append"}],
ctx=ctx,
wait=False,
)
assert result["created"] == [page]
assert result["updated"] == []
assert vfs.files[page] == "initial body"
@pytest.mark.asyncio
async def test_batch_resource_files_compile_independent_rnfv_plans(
monkeypatch,
_coordinator_queue_for_unit_tests,
):
root = "viking://resources/wiki"
first = f"{root}/docs/a.md"
second = f"{root}/docs/b.md"
vfs = _VFS(root, {first: "old A"})
class _VikingDB:
def __init__(self):
self.calls = []
async def get_incremental_inventory_by_uris(self, uris, **kwargs):
self.calls.append((list(uris), kwargs))
return {
uri: {
"a-l2": {
"id": "a-l2",
"uri": uri,
"level": 2,
"md5": "old-md5",
"abstract": "old abstract",
}
}
for uri in uris
}
vikingdb = _VikingDB()
coordinator = ContentWriteCoordinator(vfs, vikingdb=vikingdb)
snapshots = []
commits = []
direct_actions = []
refresh_calls = []
async def build_snapshot(**kwargs):
snapshots.append(kwargs)
return SimpleNamespace(
request=kwargs["request_intent"],
vectors=SimpleNamespace(records_by_id={}),
)
async def build_plan(**kwargs):
uri = kwargs["request_intent"].target_uri
return ContextUpdatePlan(
root_uri=uri,
context_type="resource",
content_tree_actions=(
ContentTreeAction(
"upsert",
"",
old_kind="file",
new_kind="file",
artifact_path="",
md5="planned-md5",
),
),
file_refresh=FileRefreshIntent(file_uri=uri, md5="planned-md5"),
)
async def commit(actions, **kwargs):
assert vfs._async_agfs.held is True
commits.append((actions, kwargs))
async def enqueue(actions, **kwargs):
direct_actions.extend(actions)
return False
async def grouped_refresh(_self, **kwargs):
assert vfs._async_agfs.held is True
refresh_calls.append(kwargs["changes"])
return {
"status": "success",
"enqueued_count": 1,
"parent_actions": {f"{root}/docs": "refresh_now"},
}
monkeypatch.setattr(content_write_module, "build_rnfv_snapshot", build_snapshot)
monkeypatch.setattr(coordinator, "_build_deferred_batch_file_plan", build_plan)
monkeypatch.setattr(content_write_module, "execute_content_tree_actions", commit)
monkeypatch.setattr(content_write_module, "enqueue_direct_index_actions", enqueue)
monkeypatch.setattr(content_write_module.Summarizer, "refresh_file_parents", grouped_refresh)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
result = await coordinator.batch_write(
root_uri=root,
operations=[
{"uri": first, "content": "A", "mode": "replace"},
{"uri": second, "content": "B", "mode": "upsert"},
],
ctx=ctx,
wait=False,
)
assert result["updated"] == [first]
assert result["created"] == [second]
assert vikingdb.calls[0][0] == [first]
assert len(snapshots) == 2
assert all(snapshot["root_is_file"] is True for snapshot in snapshots)
assert all(snapshot["vector_scope"] == "self" for snapshot in snapshots)
snapshots_by_uri = {snapshot["target_uri"]: snapshot for snapshot in snapshots}
assert snapshots_by_uri[first]["vector_inventory"]["a-l2"]["uri"] == first
assert snapshots_by_uri[second]["vector_inventory"] == {}
assert len(commits) == 2
assert direct_actions == []
assert refresh_calls == []
refresh = _last_content_write_refresh(_coordinator_queue_for_unit_tests)
plans = [ContextUpdatePlan.from_dict(item["plan"]) for item in refresh["resource_plans"]]
assert {plan.file_refresh.file_uri for plan in plans if plan.file_refresh is not None} == {
first,
second,
}
assert {
item["plan"]["file_refresh"]["file_uri"]: item["existed"]
for item in refresh["resource_plans"]
} == {
first: True,
second: False,
}
@pytest.mark.asyncio
async def test_batch_existing_files_defer_full_vector_reads_to_embedding_worker(monkeypatch):
root = "viking://resources/wiki"
operations = [
{"uri": f"{root}/docs/{index}.md", "content": f"new {index}", "mode": "replace"}
for index in range(64)
]
vfs = _VFS(
root, {operation["uri"]: f"old {index}" for index, operation in enumerate(operations)}
)
class _VikingDB:
def __init__(self):
self.inventory_calls = []
self.hydrate_incremental_records = AsyncMock(
side_effect=AssertionError("batch preparation must not hydrate full vector records")
)
async def get_incremental_inventory_by_uris(self, uris, **kwargs):
self.inventory_calls.append((list(uris), kwargs))
return {
uri: {
f"{index}-l2": {
"id": f"{index}-l2",
"uri": uri,
"level": 2,
"md5": content_md5(f"old {index}".encode()),
"abstract": f"old abstract {index}",
}
}
for index, uri in enumerate(uris)
}
vikingdb = _VikingDB()
coordinator = ContentWriteCoordinator(vfs, vikingdb=vikingdb)
monkeypatch.setattr(
content_write_module.Summarizer,
"refresh_file_parents",
AsyncMock(
return_value={"enqueued_count": 1, "parent_actions": {f"{root}/docs": "refresh_now"}}
),
)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
result = await coordinator.batch_write(
root_uri=root,
operations=operations,
ctx=ctx,
wait=False,
processing_mode="semantic_and_vectors",
)
assert result["updated"] == sorted(operation["uri"] for operation in operations)
assert len(vikingdb.inventory_calls) == 1
assert vikingdb.inventory_calls[0][0] == sorted(operation["uri"] for operation in operations)
vikingdb.hydrate_incremental_records.assert_not_awaited()
@pytest.mark.asyncio
async def test_batch_same_content_with_missing_abstract_enqueues_file_refresh(
monkeypatch,
_coordinator_queue_for_unit_tests,
):
root = "viking://resources/wiki"
uri = f"{root}/docs/a.md"
vfs = _VFS(root, {uri: "unchanged"})
class _VikingDB:
async def get_incremental_inventory_by_uris(self, uris, **kwargs):
del kwargs
return {
item: {
"a-l2": {
"id": "a-l2",
"uri": item,
"level": 2,
"md5": content_md5(b"unchanged"),
"abstract": "",
}
}
for item in uris
}
refresh_file_parents = AsyncMock(
return_value={"enqueued_count": 1, "parent_actions": {f"{root}/docs": "refresh_now"}}
)
monkeypatch.setattr(
content_write_module.Summarizer,
"refresh_file_parents",
refresh_file_parents,
)
coordinator = ContentWriteCoordinator(vfs, vikingdb=_VikingDB())
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
await coordinator.batch_write(
root_uri=root,
operations=[{"uri": uri, "content": "unchanged", "mode": "replace"}],
ctx=ctx,
wait=False,
processing_mode="semantic_and_vectors",
)
refresh_file_parents.assert_not_awaited()
refresh = _last_content_write_refresh(_coordinator_queue_for_unit_tests)
plan = ContextUpdatePlan.from_dict(refresh["resource_plans"][0]["plan"])
assert (plan.file_refresh.file_uri, plan.file_refresh.md5) == (
uri,
content_md5(b"unchanged"),
)
@pytest.mark.asyncio
async def test_batch_vectors_only_existing_file_enqueues_merge_patch_without_hydration(
monkeypatch,
_coordinator_queue_for_unit_tests,
):
root = "viking://resources/wiki"
uri = f"{root}/a.md"
vfs = _VFS(root, {uri: "old"})
class _VikingDB:
hydrate_incremental_records = AsyncMock(
side_effect=AssertionError("batch preparation must not hydrate full vector records")
)
async def get_incremental_inventory_by_uris(self, uris, **kwargs):
del kwargs
return {
uri: {
"existing-l2": {
"id": "existing-l2",
"uri": uri,
"level": 2,
"md5": content_md5(b"old"),
"abstract": "old abstract",
}
}
for uri in uris
}
actions = []
async def enqueue(direct_actions, **kwargs):
del kwargs
actions.extend(direct_actions)
return True
monkeypatch.setattr(content_write_module, "enqueue_direct_index_actions", enqueue)
coordinator = ContentWriteCoordinator(vfs, vikingdb=_VikingDB())
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
await coordinator.batch_write(
root_uri=root,
operations=[
{
"uri": uri,
"content": "new",
"tags": ["team=search"],
"tag_mode": "append",
}
],
ctx=ctx,
wait=False,
processing_mode="vectors_only",
)
assert actions == []
refresh = _last_content_write_refresh(_coordinator_queue_for_unit_tests)
plan = ContextUpdatePlan.from_dict(refresh["resource_plans"][0]["plan"])
assert len(plan.direct_index_actions) == 1
action = plan.direct_index_actions[0]
assert action.action.value == "merge"
assert action.record_id == "existing-l2"
assert action.field_patch is not None
assert action.field_patch.values == {"search_tags": ["team=search"]}
assert action.field_patch.modes == {"search_tags": "append"}
@pytest.mark.asyncio
async def test_batch_resource_plan_uses_operation_processing_mode_and_tags(monkeypatch):
root = "viking://resources/wiki"
uri = f"{root}/a.md"
vfs = _VFS(root, {uri: "old"})
class _VikingDB:
async def get_incremental_inventory_by_uris(self, uris, **kwargs):
del kwargs
return {item: {} for item in uris}
coordinator = ContentWriteCoordinator(vfs, vikingdb=_VikingDB())
seen = {}
async def build_snapshot(**kwargs):
seen["request"] = kwargs["request_intent"]
return SimpleNamespace(
request=kwargs["request_intent"], vectors=SimpleNamespace(records_by_id={})
)
async def build_plan(**kwargs):
del kwargs
return ContextUpdatePlan(uri, "resource")
monkeypatch.setattr(content_write_module, "build_rnfv_snapshot", build_snapshot)
monkeypatch.setattr(coordinator, "_build_deferred_batch_file_plan", build_plan)
monkeypatch.setattr(coordinator, "_refresh_batch", AsyncMock(return_value=None))
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
await coordinator.batch_write(
root_uri=root,
operations=[
{
"uri": uri,
"content": "new",
"mode": "replace",
"tags": ["team=search"],
"tag_mode": "append",
}
],
ctx=ctx,
wait=False,
processing_mode="vectors_only",
)
request = seen["request"]
assert request.processing_mode == "vectors_only"
assert request.scalar_intents[0].value == ("team=search",)
assert request.scalar_intents[0].mode == "append"
@pytest.mark.asyncio
async def test_batch_resource_plan_allows_clear_tags_without_tags(monkeypatch):
root = "viking://resources/wiki"
uri = f"{root}/a.md"
vfs = _VFS(root, {uri: "old"})
class _VikingDB:
async def get_incremental_inventory_by_uris(self, uris, **kwargs):
del kwargs
return {item: {} for item in uris}
coordinator = ContentWriteCoordinator(vfs, vikingdb=_VikingDB())
seen = {}
async def build_snapshot(**kwargs):
seen["request"] = kwargs["request_intent"]
return SimpleNamespace(
request=kwargs["request_intent"], vectors=SimpleNamespace(records_by_id={})
)
async def build_plan(**kwargs):
del kwargs
return ContextUpdatePlan(uri, "resource")
monkeypatch.setattr(content_write_module, "build_rnfv_snapshot", build_snapshot)
monkeypatch.setattr(coordinator, "_build_deferred_batch_file_plan", build_plan)
monkeypatch.setattr(coordinator, "_refresh_batch", AsyncMock(return_value=None))
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
await coordinator.batch_write(
root_uri=root,
operations=[{"uri": uri, "content": "new", "tag_mode": "clear"}],
ctx=ctx,
wait=False,
)
assert seen["request"].scalar_intents[0].value == ()
# The public ``clear`` spelling reaches RNFV as its backend-compatible
# equivalent: replace the scalar field with an empty tag list.
assert seen["request"].scalar_intents[0].mode == "replace"
@pytest.mark.asyncio
async def test_batch_applies_operation_acl_and_forwards_file_options(
monkeypatch,
_coordinator_queue_for_unit_tests,
):
root = "viking://resources/wiki"
uri = f"{root}/a.md"
vfs = _VFS(root)
prepared_update = AclUpdate(
uri=uri,
acl=AclSpec(acl_mode="restricted", entries=()),
)
vfs.prepare_acl_update = AsyncMock(return_value=prepared_update)
vfs.acl_manager = SimpleNamespace(apply_indexed_update=AsyncMock())
coordinator = ContentWriteCoordinator(vfs)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
await coordinator.batch_write(
root_uri=root,
operations=[
{
"uri": uri,
"content": "new",
"tags": ["team=search"],
"tag_mode": "append",
"acl": {"acl_mode": "restricted", "entries": []},
}
],
ctx=ctx,
wait=False,
)
vfs.prepare_acl_update.assert_awaited_once()
vfs.acl_manager.apply_indexed_update.assert_awaited_once_with(prepared_update, ctx)
file_options = _last_content_write_refresh(_coordinator_queue_for_unit_tests)[
"file_ingest_options"
]
assert file_options[uri]["search_tags"] == ["team=search"]
assert file_options[uri]["search_tag_mode"] == "append"
assert file_options[uri]["acl_update"] == prepared_update.model_dump(mode="json")
@pytest.mark.asyncio
async def test_batch_same_content_with_tags_is_not_skipped_as_a_conflict_noop(
monkeypatch,
_coordinator_queue_for_unit_tests,
):
root = "viking://resources/wiki"
uri = f"{root}/a.md"
vfs = _VFS(root, {uri: "same"})
coordinator = ContentWriteCoordinator(vfs)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
await coordinator.batch_write(
root_uri=root,
operations=[
{
"uri": uri,
"content": "same",
"expected_sha256": hashlib.sha256(b"same").hexdigest(),
"tags": ["team=search"],
}
],
ctx=ctx,
wait=False,
skip_conflicts=True,
)
assert vfs.writes == [uri]
refresh = _last_content_write_refresh(_coordinator_queue_for_unit_tests)
assert refresh["file_ingest_options"][uri]["search_tags"] == ["team=search"]
@pytest.mark.asyncio
async def test_batch_hands_off_file_locks_after_one_aggregated_refresh(
_coordinator_queue_for_unit_tests,
):
root = "viking://resources/wiki"
a = f"{root}/a.md"
b = f"{root}/b.md"
vfs = _VFS(root)
locks = vfs._async_agfs
coordinator = ContentWriteCoordinator(vfs)
telemetry = OperationTelemetry(operation="content.batch_write", enabled=False)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
with bind_telemetry(telemetry):
result = await coordinator.batch_write(
root_uri=root,
operations=[
{"uri": b, "content": "B", "mode": "upsert"},
{"uri": a, "content": "A", "mode": "upsert"},
],
ctx=ctx,
wait=False,
)
assert vfs.writes == [a, b]
assert result["created"] == [a, b]
assert result["queue_status"] is None
assert result["semantic_status"] == "queued"
assert result["vector_status"] == "queued"
assert locks.releases == 0
assert locks.handoffs == 1
payload = _coordinator_queue_for_unit_tests.content_write.enqueue.await_args.args[0]
assert payload["operation"] == "content_batch_write"
assert payload["resource_id"] == root
assert payload["result"]["created"] == [a, b]
assert payload["refresh"]["kind"] == "batch"
assert payload["refresh"]["refresh_kinds"] == {a: "added", b: "added"}
assert payload["lock_handoff"]["owner_id"] == "lock-1"
@pytest.mark.asyncio
async def test_batch_retries_after_root_enqueue_failure(
_coordinator_queue_for_unit_tests,
):
root = "viking://resources/wiki"
target = f"{root}/a.md"
vfs = _VFS(root)
coordinator = ContentWriteCoordinator(vfs)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
_coordinator_queue_for_unit_tests.content_write.enqueue.side_effect = RuntimeError(
"queue unavailable"
)
with pytest.raises(RuntimeError, match="queue unavailable"):
await coordinator.batch_write(
root_uri=root,
operations=[{"uri": target, "content": "A", "mode": "upsert"}],
ctx=ctx,
wait=False,
)
_coordinator_queue_for_unit_tests.content_write.enqueue.side_effect = None
result = await coordinator.batch_write(
root_uri=root,
operations=[{"uri": target, "content": "B", "mode": "upsert"}],
ctx=ctx,
wait=False,
)
assert result["updated"] == [target]
assert vfs.files[target] == "B"
@pytest.mark.asyncio
async def test_batch_cancellation_waits_for_root_enqueue_settlement(
_coordinator_queue_for_unit_tests,
):
root = "viking://resources/wiki"
target = f"{root}/a.md"
vfs = _VFS(root)
coordinator = ContentWriteCoordinator(vfs)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
enqueue_started = asyncio.Event()
finish_enqueue = asyncio.Event()
async def enqueue(_payload):
enqueue_started.set()
await finish_enqueue.wait()
return EnqueueReceipt("queued", "content-write-work-1")
_coordinator_queue_for_unit_tests.content_write.enqueue.side_effect = enqueue
request = asyncio.create_task(
coordinator.batch_write(
root_uri=root,
operations=[{"uri": target, "content": "A", "mode": "upsert"}],
ctx=ctx,
wait=False,
)
)
await enqueue_started.wait()
request.cancel()
await asyncio.sleep(0)
assert not request.done()
finish_enqueue.set()
with pytest.raises(asyncio.CancelledError):
await request
assert vfs._async_agfs.handoffs == 1
assert vfs._async_agfs.releases == 0
assert vfs._async_agfs.held is False
@pytest.mark.asyncio
async def test_batch_passes_final_md5s_and_old_abstracts_to_refresh(
_coordinator_queue_for_unit_tests,
):
root = "viking://resources/wiki"
existing = f"{root}/existing.md"
created = f"{root}/new.md"
vfs = _VFS(root, {existing: "old"})
class _VikingDB:
async def get_l2_diff_records_by_uris(self, uris, *, ctx):
assert uris == [existing]
return {existing: {"abstract": "old abstract", "md5": "old-md5"}}
coordinator = ContentWriteCoordinator(vfs, vikingdb=_VikingDB())
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
await coordinator.batch_write(
root_uri=root,
operations=[
{"uri": existing, "content": "new", "mode": "replace"},
{"uri": created, "content": "created", "mode": "create"},
],
ctx=ctx,
wait=False,
)
refresh = _last_content_write_refresh(_coordinator_queue_for_unit_tests)
assert refresh["file_md5s"] == {
existing: content_md5(b"new"),
created: content_md5(b"created"),
}
assert refresh["file_abstracts"] == {existing: "old abstract"}
@pytest.mark.asyncio
async def test_batch_replace_memory_preserves_metadata(monkeypatch):
root = "viking://user/default/memories/preferences"
memory_uri = f"{root}/theme.md"
metadata = {
"tags": ["ui", "preference"],
"fields": {"topic": "theme"},
"version": 7,
}
original = MemoryFileUtils.write(
MemoryFile(content="Original preference", extra_fields=metadata)
)
expected = MemoryFileUtils.read(original, uri=memory_uri)
vfs = _VFS(root, {memory_uri: original})
coordinator = ContentWriteCoordinator(vfs)
async def refresh(**kwargs):
del kwargs
return None
monkeypatch.setattr(coordinator, "_refresh_batch", refresh)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
result = await coordinator.batch_write(
root_uri=root,
operations=[
{
"uri": memory_uri,
"content": "Updated preference",
"mode": "replace",
}
],
ctx=ctx,
wait=False,
)
stored = MemoryFileUtils.read(vfs.files[memory_uri], uri=memory_uri)
assert result["updated"] == [memory_uri]
assert stored.content == "Updated preference"
assert stored.extra_fields == expected.extra_fields
@pytest.mark.asyncio
async def test_batch_reports_skipped_directory_and_queued_file_vectors(monkeypatch):
root = "viking://resources/wide"
page = f"{root}/page.md"
coordinator = ContentWriteCoordinator(_VFS(root))
async def enqueue(**kwargs):
del kwargs
return FreshnessAction.NOOP
monkeypatch.setattr(coordinator, "_enqueue_semantic_refresh_changes", enqueue)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
result = await coordinator.batch_write(
root_uri=root,
operations=[
{
"uri": page,
"content": "content",
"mode": "create",
}
],
ctx=ctx,
wait=False,
)
assert result["semantic_status"] == "queued"
assert result["vector_status"] == "queued"
assert result["queue_status"] is None
@pytest.mark.asyncio
async def test_batch_upserts_binary_content(_coordinator_queue_for_unit_tests):
root = "viking://resources/wiki"
image = f"{root}/figure.png"
original = b"\x89PNG\r\n\x1a\nold"
replacement = b"\x89PNG\r\n\x1a\nnew"
vfs = _VFS(root, {image: original})
coordinator = ContentWriteCoordinator(vfs)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
operation = {
"uri": image,
"content_base64": base64.b64encode(replacement).decode(),
"mode": "upsert",
}
result = await coordinator.batch_write(
root_uri=root, operations=[operation], ctx=ctx, wait=False
)
assert result["updated"] == [image]
assert vfs.files[image] == replacement
refresh = _last_content_write_refresh(_coordinator_queue_for_unit_tests)
assert refresh["file_md5s"] == {image: content_md5(replacement)}
retry = await coordinator.batch_write(
root_uri=root, operations=[operation], ctx=ctx, wait=False
)
assert retry["updated"] == [image]
@pytest.mark.asyncio
async def test_batch_rejects_invalid_binary_payload_and_memory_binary():
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
resource_root = "viking://resources/wiki"
coordinator = ContentWriteCoordinator(_VFS(resource_root))
with pytest.raises(InvalidArgumentError, match="content_base64 is invalid"):
await coordinator.batch_write(
root_uri=resource_root,
operations=[
{
"uri": f"{resource_root}/figure.png",
"content_base64": "not base64!",
"mode": "upsert",
}
],
ctx=ctx,
wait=False,
)
memory_root = "viking://user/default/memories/preferences/wiki"
coordinator = ContentWriteCoordinator(_VFS(memory_root))
with pytest.raises(InvalidArgumentError, match="not supported for memories"):
await coordinator.batch_write(
root_uri=memory_root,
operations=[
{
"uri": f"{memory_root}/figure.png",
"content_base64": base64.b64encode(b"png").decode(),
"mode": "upsert",
}
],
ctx=ctx,
wait=False,
)
@pytest.mark.asyncio
async def test_batch_partial_failure_refreshes_successful_files_and_retry_is_safe(
_coordinator_queue_for_unit_tests,
):
root = "viking://resources/wiki"
a = f"{root}/a.md"
b = f"{root}/b.md"
vfs = _VFS(root, fail_uri=b)
coordinator = ContentWriteCoordinator(vfs)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
operations = [
{"uri": a, "content": "A", "mode": "upsert"},
{"uri": b, "content": "B", "mode": "upsert"},
]
with pytest.raises(OSError, match="injected"):
await coordinator.batch_write(root_uri=root, operations=operations, ctx=ctx, wait=False)
payload = _last_content_write_payload(_coordinator_queue_for_unit_tests)
assert payload["failure"]["message"] == "injected write failure"
assert payload["refresh"]["refresh_kinds"] == {a: "added"}
vfs.fail_uri = None
result = await coordinator.batch_write(
root_uri=root, operations=operations, ctx=ctx, wait=False
)
assert result["updated"] == [a]
assert result["created"] == [b]
refresh = _last_content_write_refresh(_coordinator_queue_for_unit_tests)
assert refresh["refresh_kinds"] == {a: "modified", b: "added"}
@pytest.mark.asyncio
async def test_batch_refresh_groups_resource_and_memory_work(monkeypatch):
coordinator = ContentWriteCoordinator(_VFS("viking://resources/wiki"), vikingdb=object())
semantic_calls = []
overview_calls = []
embedding_calls = []
async def resolve_root(uri, **kwargs):
del uri, kwargs
return "viking://resources/wiki"
async def enqueue(**kwargs):
semantic_calls.append(kwargs)
return FreshnessAction.MARK_PENDING
async def overview(**kwargs):
overview_calls.append(kwargs)
async def embedding(**kwargs):
embedding_calls.append(kwargs)
return False
monkeypatch.setattr(coordinator, "_resolve_root_uri", resolve_root)
monkeypatch.setattr(coordinator, "_enqueue_semantic_refresh_changes", enqueue)
monkeypatch.setattr(content_write_module.MemoryUpdater, "refresh_schema_overview", overview)
monkeypatch.setattr(content_write_module.MemoryUpdater, "refresh_file_embedding", embedding)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
outcome = await coordinator._refresh_batch(
refresh_kinds={
"viking://resources/wiki/a.md": "added",
"viking://resources/wiki/b.md": "modified",
"viking://user/default/memories/preferences/wiki/a.md": "added",
"viking://user/default/memories/preferences/wiki/b.md": "modified",
},
ctx=ctx,
wait=False,
telemetry_id="",
)
assert len(semantic_calls) == 1
assert semantic_calls[0]["changes"] == {
"added": ["viking://resources/wiki/a.md"],
"modified": ["viking://resources/wiki/b.md"],
}
assert len(overview_calls) == 1
assert overview_calls[0]["strict"] is True
assert len(embedding_calls) == 2
assert outcome.statuses(wait=False) == ("deferred", "queued")
assert all(call["strict"] is True for call in embedding_calls)
@pytest.mark.asyncio
async def test_batch_write_api_upserts_files(client_with_resource):
client, root = client_with_resource
listing = await client.get(
"/api/v1/fs/ls",
params={"uri": root, "simple": True, "recursive": True},
)
existing = listing.json()["result"][0]
created = f"{root}/compile-batch-created.md"
operations = [
{
"uri": existing,
"content": "# Batch updated",
"mode": "upsert",
},
{
"uri": created,
"content": "# Batch created",
"mode": "upsert",
},
]
first = await client.post(
"/api/v1/content/batch-write",
json={"root_uri": root, "operations": operations, "wait": False},
)
assert first.status_code == 200
assert first.json()["result"]["updated"] == [existing]
assert first.json()["result"]["created"] == [created]
retry = await client.post(
"/api/v1/content/batch-write",
json={"root_uri": root, "operations": operations, "wait": False},
)
assert retry.status_code == 409
waited = await client.post(
"/api/v1/system/wait",
json={},
)
assert waited.status_code == 200
retry = await client.post(
"/api/v1/content/batch-write",
json={"root_uri": root, "operations": operations, "wait": False},
)
assert retry.status_code == 200
assert retry.json()["result"]["updated"] == sorted([existing, created])
@pytest.mark.asyncio
@pytest.mark.parametrize(
("filename", "content"),
[
("compile-figure.png", b"\x89PNG\r\n\x1a\ncompile"),
("compile-report.pdf", b"%PDF-1.7\ncompile"),
],
)
async def test_batch_write_api_creates_binary_file(client_with_resource, filename, content):
client, root = client_with_resource
artifact = f"{root}/{filename}"
response = await client.post(
"/api/v1/content/batch-write",
json={
"root_uri": root,
"wait": False,
"operations": [
{
"uri": artifact,
"content_base64": base64.b64encode(content).decode(),
"mode": "upsert",
}
],
},
)
assert response.status_code == 200
assert response.json()["result"]["created"] == [artifact]
downloaded = await client.get("/api/v1/content/download", params={"uri": artifact})
assert downloaded.status_code == 200
assert downloaded.content == content
@pytest.mark.asyncio
async def test_batch_write_api_create_overwrites_and_batch_continues(client_with_resource):
client, root = client_with_resource
listing = await client.get(
"/api/v1/fs/ls",
params={"uri": root, "simple": True, "recursive": True},
)
existing = listing.json()["result"][0]
created = f"{root}/compile-created-by-upsert.md"
response = await client.post(
"/api/v1/content/batch-write",
json={
"root_uri": root,
"wait": False,
"operations": [
{
"uri": existing,
"content": "create aliases replace",
"mode": "create",
},
{
"uri": created,
"content": "created by upsert",
"mode": "upsert",
},
],
},
)
assert response.status_code == 200
result = response.json()["result"]
assert result["updated"] == [existing]
assert result["created"] == [created]
existing_body = await client.get("/api/v1/content/read", params={"uri": existing, "raw": True})
assert existing_body.status_code == 200
assert existing_body.json()["result"] == "create aliases replace"
created_body = await client.get("/api/v1/content/read", params={"uri": created, "raw": True})
assert created_body.status_code == 200
assert created_body.json()["result"] == "created by upsert"
@pytest.mark.asyncio
async def test_batch_write_api_missing_append_creates_file(client_with_resource):
client, root = client_with_resource
target = f"{root}/compile-created-by-append.md"
response = await client.post(
"/api/v1/content/batch-write",
json={
"root_uri": root,
"wait": False,
"operations": [
{
"uri": target,
"content": "initial append body",
"mode": "append",
}
],
},
)
assert response.status_code == 200
assert response.json()["result"]["created"] == [target]
body = await client.get("/api/v1/content/read", params={"uri": target, "raw": True})
assert body.status_code == 200
assert body.json()["result"] == "initial append body"
@pytest.mark.asyncio
async def test_batch_write_rejects_path_traversal(client_with_resource):
client, root = client_with_resource
response = await client.post(
"/api/v1/content/batch-write",
json={
"root_uri": root,
"wait": False,
"operations": [
{
"uri": f"{root}/../escaped.md",
"content": "escape",
"mode": "upsert",
}
],
},
)
assert response.status_code == 400
assert response.json()["error"]["code"] == "INVALID_ARGUMENT"