Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(storage): 保持写入锁直到异步索引完成 (#6016)

hezhang-12166c802cd9469b
diff --git a/openviking/storage/content_write.py b/openviking/storage/content_write.py
index 362c73e58..a6d8cfa14 100644
--- a/openviking/storage/content_write.py
+++ b/openviking/storage/content_write.py
@@ -327,7 +327,7 @@ class ContentWriteCoordinator:
prepared_resources: dict[str, _PreparedInlineResource] = {}
committed_resource_plans: list[_PreparedInlineResource] = []
write_error: Exception | None = None
- lock_released = False
+ lock_settled = False
try:
for operation in normalized_operations:
uri = operation["uri"]
@@ -445,19 +445,31 @@ class ContentWriteCoordinator:
created.append(uri)
if prepared is None:
refresh_kinds[uri] = "added"
- finally:
+ except BaseException:
await self._viking_fs._async_agfs.pathlock_release(lease)
- lock_released = True
+ lock_settled = True
+ raise
- assert lock_released
telemetry_id = get_current_telemetry().telemetry_id
request_registered = False
+ enqueue_barrier_id: str | None = None
+ handoff_pending = False
refresh_outcome: Optional[_BatchRefreshOutcome] = None
try:
if refresh_kinds or sidecar_directories or committed_resource_plans:
- if wait and telemetry_id:
- get_request_wait_tracker().register_request(telemetry_id)
+ if telemetry_id:
+ tracker = get_request_wait_tracker()
+ tracker.register_request(telemetry_id)
request_registered = True
+ lock_handoff = await self._viking_fs._async_agfs.pathlock_to_handoff(lease)
+ _, enqueue_barrier_id = await self._enqueue_lock_coordinator(
+ root_uri=normalized_root,
+ context_type=context_type_for_uri(normalized_root),
+ ctx=ctx,
+ telemetry_id=telemetry_id,
+ lock_handoff=lock_handoff,
+ )
+ handoff_pending = True
try:
for directory_uri, options in sorted(sidecar_directories.items()):
await self._vectorize_semantic_directory(
@@ -525,19 +537,12 @@ class ContentWriteCoordinator:
if isinstance(legacy_refresh_result, _BatchRefreshOutcome)
else _BatchRefreshOutcome(queue_status=None)
)
- queue_status = (
- await self._wait_for_request(telemetry_id=telemetry_id, timeout=timeout)
- if wait and request_registered
- else legacy_outcome.queue_status
- )
- if wait and queue_status is not None:
- self._raise_refresh_errors(queue_status)
planned_actions = tuple(
FreshnessAction(action)
for action in parent_result["parent_actions"].values()
)
refresh_outcome = _BatchRefreshOutcome(
- queue_status=queue_status,
+ queue_status=legacy_outcome.queue_status,
semantic_actions=planned_actions + legacy_outcome.semantic_actions,
memory_refreshed=legacy_outcome.memory_refreshed,
embedding_requested=(
@@ -569,9 +574,57 @@ class ContentWriteCoordinator:
"cause": cause,
},
) from exc
+ finally:
+ if enqueue_barrier_id is not None:
+ get_request_wait_tracker().mark_semantic_done(
+ telemetry_id,
+ enqueue_barrier_id,
+ processed_delta=0,
+ )
+ enqueue_barrier_id = None
+
+ if handoff_pending:
+ try:
+ await self._viking_fs._async_agfs.pathlock_handoff(lease)
+ except BaseException:
+ await self._viking_fs._async_agfs.pathlock_release(lease)
+ lock_settled = True
+ raise
+ else:
+ await self._viking_fs._async_agfs.pathlock_release(lease)
+ lock_settled = True
+ queue_status = (
+ await self._wait_for_request(telemetry_id=telemetry_id, timeout=timeout)
+ if wait and request_registered
+ else refresh_outcome.queue_status
+ if refresh_outcome is not None
+ else None
+ )
+ if wait and queue_status is not None and write_error is None:
+ self._raise_refresh_errors(queue_status)
+ if refresh_outcome is not None and queue_status is not None:
+ refresh_outcome = replace(refresh_outcome, queue_status=queue_status)
else:
queue_status = None
+ await self._viking_fs._async_agfs.pathlock_release(lease)
+ lock_settled = True
finally:
+ if enqueue_barrier_id is not None:
+ get_request_wait_tracker().mark_semantic_done(
+ telemetry_id,
+ enqueue_barrier_id,
+ processed_delta=0,
+ )
+ if not lock_settled:
+ if handoff_pending:
+ try:
+ await self._viking_fs._async_agfs.pathlock_handoff(lease)
+ except BaseException:
+ await self._viking_fs._async_agfs.pathlock_release(lease)
+ raise
+ else:
+ await self._viking_fs._async_agfs.pathlock_release(lease)
+ lock_settled = True
if request_registered:
get_request_wait_tracker().cleanup(telemetry_id)
@@ -897,6 +950,47 @@ class ContentWriteCoordinator:
raise
return action
+ async def _enqueue_lock_coordinator(
+ self,
+ *,
+ root_uri: str,
+ context_type: str,
+ ctx: RequestContext,
+ telemetry_id: str,
+ lock_handoff: dict[str, Any],
+ ) -> tuple[SemanticMsg, str]:
+ """Enqueue the sole lease owner before scheduling request work."""
+ queue_manager = get_queue_manager()
+ semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
+ msg = SemanticMsg(
+ uri=root_uri,
+ context_type=context_type,
+ recursive=False,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ group_ids=ctx.group_ids,
+ role=str(ctx.role),
+ skip_vectorization=True,
+ telemetry_id=telemetry_id,
+ lock_handoff=lock_handoff,
+ generation_trigger="content_write",
+ aggregate_directory=False,
+ propagate_to_parent=False,
+ wait_for_request_completion=True,
+ )
+ enqueue_barrier_id = f"{msg.id}:enqueue"
+ tracker = get_request_wait_tracker()
+ tracker.register_semantic_root(telemetry_id, msg.id)
+ tracker.retain_request(telemetry_id, msg.id)
+ tracker.register_semantic_root(telemetry_id, enqueue_barrier_id)
+ try:
+ await semantic_queue.enqueue(msg)
+ except BaseException as exc:
+ tracker.mark_semantic_failed(telemetry_id, enqueue_barrier_id, str(exc))
+ tracker.mark_semantic_failed(telemetry_id, msg.id, str(exc))
+ raise
+ return msg, enqueue_barrier_id
+
@staticmethod
def _raise_refresh_errors(queue_status: Dict[str, Any]) -> None:
for name in ("Semantic", "Embedding"):
@@ -1400,8 +1494,10 @@ class ContentWriteCoordinator:
) from exc
previous_content: Optional[str] = None
- lock_released = False
+ lock_settled = False
request_registered = False
+ enqueue_barrier_id: str | None = None
+ handoff_pending = False
try:
target_state = await self._classify_locked_write_target(uri, ctx=ctx)
target_preexisting = target_state.preexisting
@@ -1435,9 +1531,22 @@ class ContentWriteCoordinator:
ingest_options=ingest_options,
)
- if wait and telemetry_id:
- get_request_wait_tracker().register_request(telemetry_id)
+ has_async_work = bool(
+ prepared.plan.direct_index_actions or prepared.plan.file_refresh is not None
+ )
+ if has_async_work and telemetry_id:
+ tracker = get_request_wait_tracker()
+ tracker.register_request(telemetry_id)
request_registered = True
+ lock_handoff = await self._viking_fs._async_agfs.pathlock_to_handoff(lease)
+ _, enqueue_barrier_id = await self._enqueue_lock_coordinator(
+ root_uri=uri,
+ context_type=context_type,
+ ctx=ctx,
+ telemetry_id=telemetry_id,
+ lock_handoff=lock_handoff,
+ )
+ handoff_pending = True
work = await commit_and_enqueue_plan(
prepared.plan,
@@ -1458,8 +1567,23 @@ class ContentWriteCoordinator:
ingest_options.acl_update, ctx
)
- await self._viking_fs._async_agfs.pathlock_release(lease)
- lock_released = True
+ if enqueue_barrier_id is not None:
+ get_request_wait_tracker().mark_semantic_done(
+ telemetry_id,
+ enqueue_barrier_id,
+ processed_delta=0,
+ )
+ enqueue_barrier_id = None
+ if handoff_pending:
+ try:
+ await self._viking_fs._async_agfs.pathlock_handoff(lease)
+ except BaseException:
+ await self._viking_fs._async_agfs.pathlock_release(lease)
+ lock_settled = True
+ raise
+ else:
+ await self._viking_fs._async_agfs.pathlock_release(lease)
+ lock_settled = True
queue_status = (
await self._wait_for_request(telemetry_id=telemetry_id, timeout=timeout)
@@ -1480,15 +1604,29 @@ class ContentWriteCoordinator:
semantic_status=semantic_status,
vector_status=vector_status,
)
- except Exception:
+ except BaseException:
# Once formal content is committed it is not rolled back for a later
# enqueue failure. The request propagates that failure so the caller
# can retry; explicit writes always recommit their final bytes, even
# when N already matches V, which makes that retry safe while V lags F.
- if not lock_released:
- await self._viking_fs._async_agfs.pathlock_release(lease)
raise
finally:
+ if enqueue_barrier_id is not None:
+ get_request_wait_tracker().mark_semantic_done(
+ telemetry_id,
+ enqueue_barrier_id,
+ processed_delta=0,
+ )
+ if not lock_settled:
+ if handoff_pending:
+ try:
+ await self._viking_fs._async_agfs.pathlock_handoff(lease)
+ except BaseException:
+ await self._viking_fs._async_agfs.pathlock_release(lease)
+ raise
+ else:
+ await self._viking_fs._async_agfs.pathlock_release(lease)
+ lock_settled = True
if request_registered:
get_request_wait_tracker().cleanup(telemetry_id)
diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.py
index 21bd8bc09..f4e41d297 100644
--- a/openviking/storage/queuefs/semantic_msg.py
+++ b/openviking/storage/queuefs/semantic_msg.py
@@ -74,6 +74,10 @@ class SemanticMsg:
aggregate_directory: bool = True
use_hierarchical_aggregation: bool = False
propagate_to_parent: bool = True
+ # Hold a producer's lease until already-enqueued embedding work settles.
+ wait_for_embeddings_only: bool = False
+ # Hold a producer's lease until every other root in this request settles.
+ wait_for_request_completion: bool = False
copy_source_uri: str = ""
# Per-file md5 of final stored bytes, keyed by target URI. Supplied by the
# local incremental apply so the tree executor's re-vectorization writes a fresh
@@ -110,6 +114,8 @@ class SemanticMsg:
aggregate_directory: bool = True,
use_hierarchical_aggregation: bool = False,
propagate_to_parent: bool = True,
+ wait_for_embeddings_only: bool = False,
+ wait_for_request_completion: bool = False,
copy_source_uri: str = "",
file_md5s: Optional[Dict[str, str]] = None,
artifact_ref: Optional[Dict[str, Any]] = None,
@@ -147,6 +153,8 @@ class SemanticMsg:
self.aggregate_directory = bool(aggregate_directory)
self.use_hierarchical_aggregation = bool(use_hierarchical_aggregation)
self.propagate_to_parent = bool(propagate_to_parent)
+ self.wait_for_embeddings_only = bool(wait_for_embeddings_only)
+ self.wait_for_request_completion = bool(wait_for_request_completion)
self.copy_source_uri = copy_source_uri
self.file_md5s = dict(file_md5s or {})
self.artifact_ref = dict(artifact_ref) if artifact_ref else None
@@ -234,6 +242,8 @@ class SemanticMsg:
aggregate_directory=data.get("aggregate_directory", True),
use_hierarchical_aggregation=data.get("use_hierarchical_aggregation", False),
propagate_to_parent=data.get("propagate_to_parent", True),
+ wait_for_embeddings_only=data.get("wait_for_embeddings_only", False),
+ wait_for_request_completion=data.get("wait_for_request_completion", False),
copy_source_uri=data.get("copy_source_uri", ""),
file_md5s=(data.get("file_md5s") if isinstance(data.get("file_md5s"), dict) else None),
artifact_ref=(
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index f033bbcbe..58930f4b6 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -554,7 +554,11 @@ class SemanticProcessor(DequeueHandlerBase):
get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
return ProcessResult.success()
- if self._vlm_resolver is None:
+ if (
+ not msg.wait_for_embeddings_only
+ and not msg.wait_for_request_completion
+ and self._vlm_resolver is None
+ ):
raise RuntimeError(
"SemanticProcessor requires a VLM resolver for account-owned work"
)
@@ -563,10 +567,27 @@ class SemanticProcessor(DequeueHandlerBase):
return ProcessResult.success()
semantic_lock = work.scope
assert semantic_lock is not None
+ if (
+ msg.wait_for_request_completion
+ and not get_request_wait_tracker().is_complete_except(
+ msg.telemetry_id, msg.id
+ )
+ ):
+ # Keep a single-worker semantic queue moving while this
+ # coordinator retains the producer's lease.
+ await asyncio.sleep(0.05)
+ await work.reenqueue()
+ execute_status = "requeued"
+ return ProcessResult.requeued()
dag_stats = None
processing_succeeded = False
try:
- if msg.plan is not None:
+ if msg.wait_for_embeddings_only or msg.wait_for_request_completion:
+ # SemanticMessageWork drains embeddings for the single-write
+ # coordinator; the batch coordinator reaches this point only
+ # after every sibling root has settled.
+ pass
+ elif msg.plan is not None:
if msg.uri.rstrip("/") != msg.plan.root_uri:
raise ValueError("semantic message URI must match plan root_uri")
if msg.context_type != msg.plan.context_type:
@@ -690,7 +711,12 @@ class SemanticProcessor(DequeueHandlerBase):
incremental_update=is_incremental,
target_uri=target_uri,
recursive=msg.recursive,
- lock=semantic_lock.lock,
+ lock=(
+ None
+ if msg.generation_trigger == "content_write"
+ and msg.lock_handoff is not None
+ else semantic_lock.lock
+ ),
is_code_repo=msg.is_code_repo,
changes=changes,
skip_vectorization=msg.skip_vectorization,
@@ -734,8 +760,14 @@ class SemanticProcessor(DequeueHandlerBase):
failure = work.failure_result(dag_stats)
if failure is not None:
return failure
- get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id)
- self._merge_request_stats(msg.telemetry_id, processed=1)
+ is_request_coordinator = msg.wait_for_request_completion
+ get_request_wait_tracker().mark_semantic_done(
+ msg.telemetry_id,
+ msg.id,
+ processed_delta=0 if is_request_coordinator else 1,
+ )
+ if not is_request_coordinator:
+ self._merge_request_stats(msg.telemetry_id, processed=1)
logger.info(f"Completed semantic generation for: {msg.uri}")
breaker.record_success()
await self._cleanup_local_artifact(msg)
diff --git a/openviking/storage/queuefs/semantic_work.py b/openviking/storage/queuefs/semantic_work.py
index 65daf3124..116088eda 100644
--- a/openviking/storage/queuefs/semantic_work.py
+++ b/openviking/storage/queuefs/semantic_work.py
@@ -62,7 +62,7 @@ class SemanticMessageWork:
async def finish_processing(self, succeeded: bool) -> None:
assert self.scope is not None
- if self.msg.plan is not None and self.msg.lock_handoff is not None:
+ if self.msg.lock_handoff is not None:
await get_request_wait_tracker().wait_for_embeddings(
self.msg.telemetry_id,
stop_waiting=getattr(self.processor, "_embedding_worker_stopped", None),
diff --git a/openviking/telemetry/request_wait_tracker.py b/openviking/telemetry/request_wait_tracker.py
index 8553a543c..1b5d30b92 100644
--- a/openviking/telemetry/request_wait_tracker.py
+++ b/openviking/telemetry/request_wait_tracker.py
@@ -261,6 +261,17 @@ class RequestWaitTracker:
return True
return not state.pending_semantic_roots and not state.pending_embedding_roots
+ def is_complete_except(self, telemetry_id: str, semantic_root_id: str) -> bool:
+ """Return whether all work except one coordinating semantic root has settled."""
+ if not telemetry_id:
+ return True
+ with self._lock:
+ state = self._states.get(telemetry_id)
+ if state is None:
+ return True
+ pending_semantic = state.pending_semantic_roots - {semantic_root_id}
+ return not pending_semantic and not state.pending_embedding_roots
+
async def wait_for_request(
self,
telemetry_id: str,
diff --git a/tests/server/test_content_batch_write.py b/tests/server/test_content_batch_write.py
index 0478c656e..6190fcce0 100644
--- a/tests/server/test_content_batch_write.py
+++ b/tests/server/test_content_batch_write.py
@@ -16,6 +16,8 @@ from openviking.storage.context_update_plan import (
FileRefreshIntent,
)
from openviking.storage.queuefs.semantic_ops.freshness_policy import FreshnessAction
+from openviking.telemetry.context import bind_telemetry
+from openviking.telemetry.operation import OperationTelemetry
from openviking.utils.content_hash import content_md5
from openviking_cli.exceptions import (
InvalidArgumentError,
@@ -29,12 +31,28 @@ class _PathLockClient:
def __init__(self):
self.held = False
self.releases = 0
+ self.handoffs = 0
+ self.paths = []
async def pathlock_acquire_exact_batch(self, paths):
- del paths
+ self.paths = list(paths)
self.held = True
return {"lease_ref": "lock-1"}
+ async def pathlock_to_handoff(self, lease):
+ assert lease == {"lease_ref": "lock-1"}
+ assert self.held is True
+ return {
+ "owner_id": "lock-1",
+ "covered_paths": [{"path": path, "kind": "exact"} for path in self.paths],
+ }
+
+ async def pathlock_handoff(self, lease):
+ assert lease == {"lease_ref": "lock-1"}
+ assert self.held is True
+ self.held = False
+ self.handoffs += 1
+
async def pathlock_release(self, lease):
assert lease == {"lease_ref": "lock-1"}
self.held = False
@@ -95,6 +113,27 @@ class _VFS:
self.writes.append(uri)
[email protected](autouse=True)
+def _coordinator_queue_for_unit_tests(monkeypatch, request):
+ if "service" in request.fixturenames:
+ yield None
+ return
+
+ queue = SimpleNamespace(enqueue=AsyncMock(return_value="queued"))
+ queue_manager = SimpleNamespace(
+ SEMANTIC="semantic",
+ get_queue=lambda *args, **kwargs: queue,
+ )
+ monkeypatch.setattr(content_write_module, "get_queue_manager", lambda: queue_manager)
+ yield queue
+
+ tracker = content_write_module.get_request_wait_tracker()
+ for call in queue.enqueue.await_args_list:
+ msg = call.args[0]
+ if msg.wait_for_request_completion:
+ tracker.mark_semantic_done(msg.telemetry_id, msg.id, processed_delta=0)
+
+
@pytest.mark.asyncio
async def test_batch_accepts_more_than_256_operations(monkeypatch):
root = "viking://resources/wiki"
@@ -261,7 +300,7 @@ async def test_batch_resource_files_compile_independent_rnfv_plans(monkeypatch):
return False
async def grouped_refresh(_self, **kwargs):
- assert vfs._async_agfs.held is False
+ assert vfs._async_agfs.held is True
refresh_calls.append(kwargs["changes"])
return {
"status": "success",
@@ -630,7 +669,7 @@ async def test_batch_same_content_with_tags_is_not_skipped_as_a_conflict_noop(mo
@pytest.mark.asyncio
-async def test_batch_releases_file_locks_before_one_aggregated_refresh(monkeypatch):
+async def test_batch_hands_off_file_locks_after_one_aggregated_refresh(monkeypatch):
root = "viking://resources/wiki"
a = f"{root}/a.md"
b = f"{root}/b.md"
@@ -638,27 +677,43 @@ async def test_batch_releases_file_locks_before_one_aggregated_refresh(monkeypat
locks = vfs._async_agfs
coordinator = ContentWriteCoordinator(vfs)
calls = []
+ queue = SimpleNamespace(enqueue=AsyncMock(return_value="queued"))
+ queue_manager = SimpleNamespace(
+ SEMANTIC="semantic",
+ get_queue=lambda *args, **kwargs: queue,
+ )
+ telemetry = OperationTelemetry(operation="content.batch_write", enabled=False)
async def refresh(**kwargs):
- assert locks.held is False
+ assert locks.held is True
calls.append(kwargs["refresh_kinds"])
return {"Semantic": {"processed": 1, "error_count": 0}}
monkeypatch.setattr(coordinator, "_refresh_batch", refresh)
+ monkeypatch.setattr(content_write_module, "get_queue_manager", lambda: queue_manager)
ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.USER)
- result = await coordinator.batch_write(
- root_uri=root,
- operations=[
- {"uri": b, "content": "B", "mode": "upsert"},
- {"uri": a, "content": "A", "mode": "upsert"},
- ],
- ctx=ctx,
- wait=False,
- )
+ with bind_telemetry(telemetry):
+ result = await coordinator.batch_write(
+ root_uri=root,
+ operations=[
+ {"uri": b, "content": "B", "mode": "upsert"},
+ {"uri": a, "content": "A", "mode": "upsert"},
+ ],
+ ctx=ctx,
+ wait=False,
+ )
assert vfs.writes == [a, b]
assert result["created"] == [a, b]
assert calls == [{a: "added", b: "added"}]
- assert locks.releases == 1
+ assert locks.releases == 0
+ assert locks.handoffs == 1
+ coordinator_msg = queue.enqueue.await_args.args[0]
+ assert coordinator_msg.wait_for_request_completion is True
+ content_write_module.get_request_wait_tracker().mark_semantic_done(
+ telemetry.telemetry_id,
+ coordinator_msg.id,
+ processed_delta=0,
+ )
@pytest.mark.asyncio
@@ -855,7 +910,7 @@ async def test_batch_partial_failure_refreshes_successful_files_and_retry_is_saf
calls = []
async def refresh(**kwargs):
- assert locks.held is False
+ assert locks.held is True
calls.append(dict(kwargs["refresh_kinds"]))
monkeypatch.setattr(coordinator, "_refresh_batch", refresh)
@@ -997,6 +1052,18 @@ async def test_batch_write_api_upserts_files(client_with_resource):
assert first.json()["result"]["updated"] == [existing]
assert first.json()["result"]["created"] == [created]
+ retry = await client.post(
+ "/api/v1/content/batch-write",
+ json={"root_uri": root, "operations": operations, "wait": False},
+ )
+ assert retry.status_code == 409
+
+ waited = await client.post(
+ "/api/v1/system/wait",
+ json={"timeout": 30.0},
+ )
+ assert waited.status_code == 200
+
retry = await client.post(
"/api/v1/content/batch-write",
json={"root_uri": root, "operations": operations, "wait": False},
diff --git a/tests/storage/test_content_write_processing_mode.py b/tests/storage/test_content_write_processing_mode.py
index 079e82ec6..e97ef1a03 100644
--- a/tests/storage/test_content_write_processing_mode.py
+++ b/tests/storage/test_content_write_processing_mode.py
@@ -14,8 +14,10 @@ from openviking.storage.abstract_overview import (
render_abstract_overview,
)
from openviking.storage.content_write import ContentWriteCoordinator
-from openviking.storage.context_update_plan import ContextUpdatePlan
+from openviking.storage.context_update_plan import ContextUpdatePlan, FileRefreshIntent
from openviking.storage.queuefs.semantic_ops.freshness_policy import FreshnessAction
+from openviking.telemetry.context import bind_telemetry
+from openviking.telemetry.operation import OperationTelemetry
from openviking.utils.ingest_options import IngestOptions
from openviking_cli.exceptions import InvalidArgumentError, NotFoundError
from openviking_cli.session.user_id import UserIdentifier
@@ -27,6 +29,7 @@ class _FakePathLock:
def __init__(self):
self._lease = SimpleNamespace(id="lock-1")
self.release_calls = []
+ self.handoff_calls = []
async def pathlock_acquire_exact(self, lock_path):
del lock_path
@@ -35,6 +38,15 @@ class _FakePathLock:
async def pathlock_release(self, lease):
self.release_calls.append(lease.id)
+ async def pathlock_to_handoff(self, lease):
+ return {
+ "owner_id": lease.id,
+ "covered_paths": [{"path": "/fake/file", "kind": "exact"}],
+ }
+
+ async def pathlock_handoff(self, lease):
+ self.handoff_calls.append(lease.id)
+
class _FakeVikingFS:
def __init__(self):
@@ -242,6 +254,73 @@ async def test_direct_write_reads_target_once_after_lock_and_reuses_formal_state
assert captured["vector_inventory"] is None
+async def test_direct_write_hands_exact_lock_to_request_coordinator(monkeypatch, ctx):
+ uri = "viking://resources/demo.md"
+ fake_fs = _FakeVikingFS()
+ fake_fs.stat = AsyncMock(return_value={"isDir": False})
+ coordinator = ContentWriteCoordinator(viking_fs=fake_fs)
+ plan = ContextUpdatePlan(
+ uri,
+ "resource",
+ file_refresh=FileRefreshIntent(uri, "new-md5"),
+ )
+ coordinator._prepare_inline_resource_plan = AsyncMock(
+ return_value=SimpleNamespace(
+ plan=plan,
+ store=None,
+ artifact_ref=None,
+ target=None,
+ )
+ )
+ monkeypatch.setattr(
+ content_write_module,
+ "commit_and_enqueue_plan",
+ AsyncMock(
+ return_value=SimpleNamespace(
+ semantic_requested=True,
+ vector_requested=True,
+ semantic_action=FreshnessAction.REFRESH_NOW.value,
+ )
+ ),
+ )
+ queue = SimpleNamespace(enqueue=AsyncMock(return_value="queued"))
+ manager = SimpleNamespace(SEMANTIC="semantic", get_queue=lambda *args, **kwargs: queue)
+ monkeypatch.setattr(content_write_module, "get_queue_manager", lambda: manager)
+ telemetry = OperationTelemetry(operation="content.write", enabled=False)
+
+ with bind_telemetry(telemetry):
+ result = await coordinator._write_direct_with_refresh(
+ uri=uri,
+ root_uri="viking://resources",
+ content="updated",
+ mode="replace",
+ response_mode="replace",
+ context_type="resource",
+ wait=False,
+ timeout=None,
+ ctx=ctx,
+ telemetry_id=telemetry.telemetry_id,
+ ingest_options=IngestOptions(),
+ )
+
+ coordinator_msg = queue.enqueue.await_args.args[0]
+ assert coordinator_msg.wait_for_request_completion is True
+ assert coordinator_msg.lock_handoff["covered_paths"] == [
+ {"path": "/fake/file", "kind": "exact"}
+ ]
+ assert fake_fs._async_agfs.release_calls == []
+ assert fake_fs._async_agfs.handoff_calls == ["lock-1"]
+ assert result["semantic_status"] == "queued"
+ assert result["vector_status"] == "queued"
+
+ content_write_module.get_request_wait_tracker().mark_semantic_done(
+ telemetry.telemetry_id,
+ coordinator_msg.id,
+ processed_delta=0,
+ )
+
+
@pytest.mark.asyncio
async def test_direct_write_commits_final_bytes_when_rnfv_content_is_unchanged(monkeypatch, ctx):
"""An explicit write owns F even when N already matches the indexed V md5."""
diff --git a/tests/storage/test_semantic_msg_ingest_options.py b/tests/storage/test_semantic_msg_ingest_options.py
index 9e6644525..c1fe702db 100644
--- a/tests/storage/test_semantic_msg_ingest_options.py
+++ b/tests/storage/test_semantic_msg_ingest_options.py
@@ -73,6 +73,28 @@ def test_semantic_msg_round_trips_deferred_aggregation_flag():
assert SemanticMsg.from_json(msg.to_json()).aggregate_directory is False
+def test_semantic_msg_round_trips_request_completion_coordinator_flag():
+ msg = SemanticMsg(
+ uri="viking://resources/wide",
+ context_type="resource",
+ wait_for_request_completion=True,
+ )
+
+ assert SemanticMsg.from_json(msg.to_json()).wait_for_request_completion is True
+
+
+def test_semantic_msg_defaults_request_completion_coordinator_flag_for_legacy_payload():
+ msg = SemanticMsg.from_dict(
+ {
+ "uri": "viking://resources/wide",
+ "context_type": "resource",
+ "account_id": "default",
+ }
+ )
+
+ assert msg.wait_for_request_completion is False
+
+
def test_semantic_msg_round_trips_hierarchical_aggregation_policy():
msg = SemanticMsg(
uri="viking://user/alice/memories",
diff --git a/tests/storage/test_semantic_processor_lock_ownership.py b/tests/storage/test_semantic_processor_lock_ownership.py
index 322144086..0e94ba804 100644
--- a/tests/storage/test_semantic_processor_lock_ownership.py
+++ b/tests/storage/test_semantic_processor_lock_ownership.py
@@ -117,6 +117,41 @@ async def test_durable_plan_waits_for_embeddings_before_releasing_handoff_lock(m
assert pathlock.release_calls == ["durable-plan-lock"]
+async def test_content_write_waits_for_embeddings_before_releasing_handoff_lock(monkeypatch):
+ events = []
+ pathlock = _FakePathLock()
+ lease = {"id": "content-write-lock"}
+ msg = SemanticMsg(
+ uri="viking://resources/demo",
+ context_type="resource",
+ telemetry_id="content-write",
+ lock_handoff={"owner_id": "producer"},
+ generation_trigger="content_write",
+ )
+
+ class Tracker:
+ async def wait_for_embeddings(self, telemetry_id, **kwargs):
+ assert telemetry_id == msg.telemetry_id
+ events.append("embeddings-settled")
+
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_work.get_request_wait_tracker", lambda: Tracker()
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_lock.get_viking_fs",
+ lambda: _FakeVikingFS(pathlock),
+ )
+
+ work = SemanticMessageWork(SimpleNamespace(), msg, caller_lock=None)
+ work.scope = SemanticLockScope(lease, _owned=True)
+
+ await work.finish_processing(True)
+
+ assert events == ["embeddings-settled"]
+ assert pathlock.release_calls == ["content-write-lock"]
+
+
def test_semantic_tree_stats_aggregate_multiple_plans_for_one_request():
telemetry_id = "reindex-request-stats"
SemanticProcessor._cache_tree_stats(
diff --git a/tests/storage/test_semantic_processor_target_preexisting.py b/tests/storage/test_semantic_processor_target_preexisting.py
index 242c59bf6..e3dc12e4b 100644
--- a/tests/storage/test_semantic_processor_target_preexisting.py
+++ b/tests/storage/test_semantic_processor_target_preexisting.py
@@ -6,6 +6,7 @@ import pytest
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
from openviking.storage.viking_fs import SyncDiff
+from openviking.telemetry.request_wait_tracker import RequestWaitTracker
def _processor(**kwargs):
@@ -187,6 +188,178 @@ async def test_stale_content_write_keeps_file_work_without_directory_aggregation
processor._enqueue_parent_refresh.assert_not_awaited()
+async def test_content_write_handoff_is_not_reused_for_sidecar_writes(monkeypatch):
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.get_viking_fs",
+ lambda: _FakeVikingFS(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.SemanticTreeExecutor",
+ _FakeTreeExecutor,
+ )
+ scope = SimpleNamespace(lock={"id": "file-exact-lock"}, close=AsyncMock())
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.SemanticLockScope.resolve",
+ AsyncMock(return_value=scope),
+ )
+
+ _FakeTreeExecutor.calls = []
+ _FakeTreeExecutor.runs = []
+ processor = _processor()
+ processor._enqueue_parent_refresh = AsyncMock()
+ msg = SemanticMsg(
+ uri="viking://resources/wiki",
+ context_type="resource",
+ recursive=False,
+ changes={"modified": ["viking://resources/wiki/changed.md"]},
+ generation_trigger="content_write",
+ lock_handoff={"owner_id": "producer"},
+ )
+
+ await processor.on_dequeue(msg.to_dict())
+
+ assert _FakeTreeExecutor.calls[0]["lock"] is None
+ scope.close.assert_awaited_once()
+
+
+async def test_embedding_lock_wait_skips_semantic_execution(monkeypatch):
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.get_viking_fs",
+ lambda: _FakeVikingFS(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.SemanticTreeExecutor",
+ _FakeTreeExecutor,
+ )
+ scope = SimpleNamespace(lock={"id": "file-exact-lock"}, close=AsyncMock())
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.SemanticLockScope.resolve",
+ AsyncMock(return_value=scope),
+ )
+
+ _FakeTreeExecutor.calls = []
+ _FakeTreeExecutor.runs = []
+ processor = SemanticProcessor(vlm_resolver=None)
+ msg = SemanticMsg(
+ uri="viking://resources/wiki/changed.md",
+ context_type="resource",
+ recursive=False,
+ skip_vectorization=True,
+ lock_handoff={"owner_id": "producer"},
+ generation_trigger="content_write",
+ aggregate_directory=False,
+ wait_for_embeddings_only=True,
+ )
+
+ await processor.on_dequeue(msg.to_dict())
+
+ assert _FakeTreeExecutor.calls == []
+ scope.close.assert_awaited_once()
+
+
+async def test_batch_lock_coordinator_requeues_while_sibling_work_is_pending(monkeypatch):
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.get_viking_fs",
+ lambda: _FakeVikingFS(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.is_semantic_msg_stale",
+ lambda msg: False,
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.asyncio.sleep",
+ AsyncMock(),
+ )
+ tracker = RequestWaitTracker()
+ telemetry_id = "tm_batch_coordinator_requeue"
+ tracker.register_request(telemetry_id)
+ tracker.register_semantic_root(telemetry_id, "coordinator")
+ tracker.register_semantic_root(telemetry_id, "semantic-1")
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.get_request_wait_tracker",
+ lambda: tracker,
+ )
+
+ work = SimpleNamespace(
+ scope=SimpleNamespace(lock={"id": "batch-lock"}),
+ start=lambda: None,
+ resolve_telemetry=lambda: None,
+ acquire_lock=AsyncMock(return_value=True),
+ reenqueue=AsyncMock(),
+ close=AsyncMock(),
+ )
+ processor = SemanticProcessor(vlm_resolver=None)
+ monkeypatch.setattr(processor, "_message_work", lambda msg, lock=None: work)
+ msg = SemanticMsg(
+ uri="viking://resources/wiki",
+ context_type="resource",
+ telemetry_id=telemetry_id,
+ lock_handoff={"owner_id": "producer"},
+ wait_for_request_completion=True,
+ )
+ msg.id = "coordinator"
+
+ result = await processor.on_dequeue(msg.to_dict())
+
+ assert result.outcome.value == "requeued"
+ work.reenqueue.assert_awaited_once()
+ work.close.assert_awaited_once()
+ assert tracker.build_queue_status(telemetry_id)["Semantic"]["requeue_count"] == 0
+
+ tracker.mark_semantic_done(telemetry_id, "semantic-1")
+ tracker.mark_semantic_done(telemetry_id, "coordinator")
+ tracker.cleanup(telemetry_id)
+
+
+async def test_batch_lock_coordinator_releases_after_sibling_work_settles(monkeypatch):
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.get_viking_fs",
+ lambda: _FakeVikingFS(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.SemanticTreeExecutor",
+ _FakeTreeExecutor,
+ )
+ scope = SimpleNamespace(lock={"id": "batch-lock"}, close=AsyncMock())
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.SemanticLockScope.resolve",
+ AsyncMock(return_value=scope),
+ )
+ tracker = RequestWaitTracker()
+ telemetry_id = "tm_batch_coordinator_release"
+ tracker.register_request(telemetry_id)
+ tracker.register_semantic_root(telemetry_id, "coordinator")
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.get_request_wait_tracker",
+ lambda: tracker,
+ )
+
+ _FakeTreeExecutor.calls = []
+ _FakeTreeExecutor.runs = []
+ processor = SemanticProcessor(vlm_resolver=None)
+ msg = SemanticMsg(
+ uri="viking://resources/wiki",
+ context_type="resource",
+ telemetry_id=telemetry_id,
+ lock_handoff={"owner_id": "producer"},
+ wait_for_request_completion=True,
+ )
+ msg.id = "coordinator"
+
+ result = await processor.on_dequeue(msg.to_dict())
+
+ assert result.outcome.value == "success"
+ assert _FakeTreeExecutor.calls == []
+ scope.close.assert_awaited_once()
+ assert tracker.is_complete(telemetry_id) is True
+ assert tracker.build_queue_status(telemetry_id)["Semantic"]["processed"] == 0
+ tracker.cleanup(telemetry_id)
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("recursive", [False, True])
async def test_memory_reindex_uses_semantic_executor(monkeypatch, recursive):
diff --git a/tests/telemetry/test_request_wait_tracker.py b/tests/telemetry/test_request_wait_tracker.py
index 01efd2052..5401656cf 100644
--- a/tests/telemetry/test_request_wait_tracker.py
+++ b/tests/telemetry/test_request_wait_tracker.py
@@ -54,6 +54,44 @@ def test_request_wait_tracker_records_requeues():
}
+def test_request_wait_tracker_completes_except_coordinator_after_siblings_settle():
+ tracker = RequestWaitTracker()
+ telemetry_id = "tm_batch_coordinator"
+
+ tracker.register_request(telemetry_id)
+ tracker.register_semantic_root(telemetry_id, "coordinator")
+ tracker.register_semantic_root(telemetry_id, "semantic-1")
+ tracker.register_embedding_root(telemetry_id, "embedding-1")
+
+ assert tracker.is_complete_except(telemetry_id, "coordinator") is False
+
+ tracker.mark_semantic_done(telemetry_id, "semantic-1")
+ assert tracker.is_complete_except(telemetry_id, "coordinator") is False
+
+ tracker.mark_embedding_done(telemetry_id, "embedding-1")
+ assert tracker.is_complete_except(telemetry_id, "coordinator") is True
+ assert tracker.is_complete(telemetry_id) is False
+
+ tracker.mark_semantic_done(telemetry_id, "coordinator")
+ tracker.cleanup(telemetry_id)
+
+
+def test_request_wait_tracker_retains_coordinator_after_request_cleanup():
+ tracker = RequestWaitTracker()
+ telemetry_id = "tm_batch_cleanup"
+
+ tracker.register_request(telemetry_id)
+ tracker.register_semantic_root(telemetry_id, "coordinator")
+ tracker.retain_request(telemetry_id, "coordinator")
+ tracker.cleanup(telemetry_id)
+
+ assert tracker.has_request(telemetry_id) is True
+
+ tracker.mark_semantic_done(telemetry_id, "coordinator")
+
+ assert tracker.has_request(telemetry_id) is False
+
+
def test_request_wait_tracker_counts_contexts_indexed_by_request():
tracker = RequestWaitTracker()
telemetry_id = "tm_vectors"