Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(task): 修复 add-resource 任务重启后无法恢复 (#3334)

Qin Haojie1d4bf132ed11
diff --git a/openviking/service/core.py b/openviking/service/core.py
index d78ced348..0351a1039 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -28,9 +28,9 @@ from openviking.session import create_session_compressor
from openviking.storage import VikingDBManager
from openviking.storage.collection_schemas import init_context_collection
from openviking.storage.index_consistency import check_index_consistency
+from openviking.storage.queuefs.add_resource_processor import AddResourceProcessor
from openviking.storage.queuefs.queue_manager import QueueManager, init_queue_manager
from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessor
-from openviking.storage.queuefs.understanding_parse_processor import UnderstandingParseProcessor
from openviking.storage.transaction import LockManager, init_lock_manager
from openviking.storage.viking_fs import VikingFS, init_viking_fs
from openviking.utils.agfs_utils import (
@@ -416,12 +416,12 @@ class OpenVikingService:
)
if self._queue_manager:
- external_parse_processor = UnderstandingParseProcessor(
- self._resource_processor,
- resource_memory_link_service=self._resource_memory_link_service,
+ external_parse_processor = AddResourceProcessor(
+ self._resource_service,
+ asyncio.get_running_loop(),
)
self._queue_manager.get_queue(
- self._queue_manager.EXTERNAL_PARSE,
+ self._queue_manager.ADD_RESOURCE,
dequeue_handler=external_parse_processor,
allow_create=True,
)
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.py
index dce540802..44ec34c62 100644
--- a/openviking/service/resource_service.py
+++ b/openviking/service/resource_service.py
@@ -15,6 +15,7 @@ from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from uuid import uuid4
import httpx
@@ -46,7 +47,6 @@ from openviking.telemetry.resource_summary import (
build_queue_status_payload,
record_resource_wait_metrics,
register_wait_telemetry,
- summarize_queue_errors,
unregister_wait_telemetry,
)
from openviking.utils import is_git_repo_url, parse_code_hosting_url
@@ -100,6 +100,7 @@ _ADD_RESOURCE_ARGS_RESERVED_FIELDS = frozenset(
"create_parent",
"telemetry",
"request_validator",
+ "defer_post_processing",
}
)
@@ -325,7 +326,7 @@ class ResourceService:
raise NotInitializedError("VikingFS")
async def close_background_tasks(self) -> None:
- """Cancel in-flight background resource ingestion tasks during service shutdown."""
+ """Cancel in-flight connector monitoring tasks during service shutdown."""
if not self._background_tasks:
return
tasks = list(self._background_tasks)
@@ -334,6 +335,129 @@ class ResourceService:
await asyncio.gather(*tasks, return_exceptions=True)
self._background_tasks.clear()
+ async def _enqueue_add_resource_job(
+ self,
+ msg: Any,
+ *,
+ resource_lock: LockLease = NO_LOCK,
+ ) -> Any:
+ """Persist a job before its TaskRecord so a crash cannot orphan the task."""
+ from openviking.service.task_tracker import get_task_tracker
+ from openviking.storage.queuefs import QueueManager, get_queue_manager
+
+ try:
+ await get_queue_manager().enqueue(QueueManager.ADD_RESOURCE, msg.to_dict())
+ await resource_lock.handoff()
+ except BaseException:
+ await resource_lock.close()
+ raise
+
+ tracker = get_task_tracker()
+ task = await tracker.create(
+ "add_resource",
+ resource_id=msg.root_uri,
+ account_id=msg.account_id,
+ user_id=msg.user_id,
+ task_id=msg.task_id,
+ )
+ await tracker.update_stage(
+ task.task_id,
+ "queued",
+ account_id=msg.account_id,
+ user_id=msg.user_id,
+ )
+ return task
+
+ async def execute_add_resource_job(
+ self,
+ msg: Any,
+ *,
+ ctx: RequestContext,
+ resource_lock: Optional[LockLease],
+ stage_callback: Callable[[str], Any],
+ ) -> Dict[str, Any]:
+ """Execute one durable add-resource job inside its QueueFS consumer."""
+ resource_lock = resource_lock or NO_LOCK
+ if msg.prepared is None:
+ return await self.add_resource(
+ path=msg.path,
+ ctx=ctx,
+ to=msg.root_uri,
+ reason=msg.reason,
+ instruction=msg.instruction,
+ wait=True,
+ timeout=msg.timeout,
+ build_index=msg.build_index,
+ summarize=msg.summarize,
+ watch_interval=msg.watch_interval,
+ skip_watch_management=msg.skip_watch_management,
+ allow_local_path_resolution=msg.allow_local_path_resolution,
+ enforce_public_remote_targets=msg.enforce_public_remote_targets,
+ resource_lock=resource_lock,
+ stage_callback=stage_callback,
+ strict=msg.strict,
+ source_name=msg.source_name,
+ ignore_dirs=msg.ignore_dirs,
+ include=msg.include,
+ exclude=msg.exclude,
+ directly_upload_media=msg.directly_upload_media,
+ preserve_structure=msg.preserve_structure,
+ create_parent=msg.create_parent,
+ args=msg.args,
+ )
+
+ telemetry_id = get_current_telemetry().telemetry_id
+ request_wait_tracker = get_request_wait_tracker()
+ request_wait_tracker.register_request(telemetry_id)
+ try:
+ stage_result = stage_callback("processing_queue")
+ if inspect.isawaitable(stage_result):
+ await stage_result
+ result = await self._resource_processor.finish_prepared_resource(
+ msg.prepared,
+ ctx=ctx,
+ resource_lock=resource_lock,
+ summarize=msg.summarize,
+ build_index=msg.build_index,
+ )
+ await request_wait_tracker.wait_for_request(
+ telemetry_id,
+ timeout=msg.timeout,
+ )
+ status = request_wait_tracker.build_queue_status(telemetry_id)
+ result["queue_status"] = status
+ await self._link_resource_reason_memory(
+ result=result,
+ ctx=ctx,
+ reason=msg.reason,
+ source_name=msg.source_name,
+ timeout=msg.timeout,
+ )
+ return result
+ except TimeoutError as exc:
+ raise DeadlineExceededError("queue processing", msg.timeout) from exc
+ finally:
+ request_wait_tracker.cleanup(telemetry_id)
+ unregister_wait_telemetry(telemetry_id)
+
+ async def reacquire_add_resource_job_lock(
+ self,
+ root_uri: str,
+ ctx: RequestContext,
+ ) -> LockLease:
+ """Acquire a fresh lock when a recovered job's old handoff was released."""
+ if not self._resource_processor or not self._viking_fs:
+ raise NotInitializedError("ResourceProcessor")
+ from openviking.storage.transaction import get_lock_manager
+
+ dst_path = self._viking_fs._uri_to_path(root_uri, ctx=ctx)
+ return await self._resource_processor.acquire_resource_lock(
+ get_lock_manager(),
+ dst_path,
+ uri=root_uri,
+ timeout=0.0,
+ )
+
async def enqueue_git_add_resource(
self,
path: str,
@@ -365,7 +489,7 @@ class ResourceService:
create_parent=bool(kwargs.get("create_parent", False)),
)
- from openviking.service.task_tracker import get_task_tracker
+ from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
resource_lock: LockLease = NO_LOCK
try:
@@ -385,26 +509,23 @@ class ResourceService:
source_info=source_info,
)
- task_tracker = get_task_tracker()
- task = await task_tracker.create(
- "add_resource",
- resource_id=root_uri,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- await task_tracker.update_stage(
- task.task_id,
- "queued",
+ task_id = str(uuid4())
+ lock_handoff = resource_lock.to_handoff()
+ processor_args = {
+ key: value
+ for key, value in kwargs.items()
+ if key not in _ADD_RESOURCE_ARGS_RESERVED_FIELDS
+ }
+ msg = AddResourceMsg(
+ task_id=task_id,
+ path=path,
+ root_uri=root_uri,
+ telemetry_id=get_current_telemetry().telemetry_id or None,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
- )
-
- add_kwargs = dict(
- kwargs,
- path=path,
- ctx=ctx,
- to=root_uri,
- parent=None,
+ role=str(ctx.role),
+ actor_peer_id=ctx.actor_peer_id,
+ lock_handoff=lock_handoff.to_dict() if lock_handoff else None,
reason=reason,
instruction=instruction,
timeout=timeout,
@@ -414,18 +535,18 @@ class ResourceService:
skip_watch_management=skip_watch_management,
allow_local_path_resolution=allow_local_path_resolution,
enforce_public_remote_targets=enforce_public_remote_targets,
+ strict=bool(kwargs.get("strict", False)),
+ ignore_dirs=kwargs.get("ignore_dirs"),
+ include=kwargs.get("include"),
+ exclude=kwargs.get("exclude"),
+ directly_upload_media=bool(kwargs.get("directly_upload_media", True)),
+ preserve_structure=kwargs.get("preserve_structure"),
+ create_parent=bool(kwargs.get("create_parent", False)),
+ source_name=source_name,
+ args=self._sanitize_watch_processor_kwargs(processor_args),
)
- background = asyncio.create_task(
- self._run_add_resource_task(
- task.task_id,
- ctx=ctx,
- add_kwargs=add_kwargs,
- resource_lock=resource_lock,
- )
- )
+ task = await self._enqueue_add_resource_job(msg, resource_lock=resource_lock)
resource_lock = NO_LOCK
- self._background_tasks.add(background)
- background.add_done_callback(self._background_tasks.discard)
return {
"status": "success",
"root_uri": root_uri,
@@ -435,81 +556,6 @@ class ResourceService:
await resource_lock.close()
raise
- async def _run_add_resource_task(
- self,
- task_id: str,
- *,
- ctx: RequestContext,
- add_kwargs: Dict[str, Any],
- resource_lock: LockLease,
- ) -> None:
- from openviking.service.task_tracker import get_task_tracker
-
- task_tracker = get_task_tracker()
-
- async def _set_stage(stage: str) -> None:
- await task_tracker.update_stage(
- task_id,
- stage,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
-
- try:
- await task_tracker.start(
- task_id,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- stage="queued",
- )
- result = await self.add_resource(
- wait=True,
- resource_lock=resource_lock,
- stage_callback=_set_stage,
- **add_kwargs,
- )
- if result.get("status") == "error":
- errors = result.get("errors") or ["resource processing failed"]
- await task_tracker.fail(
- task_id,
- "; ".join(str(error) for error in errors),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- return
- queue_errors = summarize_queue_errors(result.get("queue_status"))
- if queue_errors:
- await task_tracker.fail(
- task_id,
- "queue processing failed: " + "; ".join(queue_errors),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- return
- await task_tracker.complete(
- task_id,
- result,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- except asyncio.CancelledError:
- await task_tracker.fail(
- task_id,
- "background resource ingestion cancelled",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- raise
- except Exception as exc:
- await task_tracker.fail(
- task_id,
- str(exc),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- finally:
- await resource_lock.close()
-
async def _plan_resource_target(
self,
*,
@@ -731,7 +777,8 @@ class ResourceService:
telemetry = get_current_telemetry()
telemetry_id = register_wait_telemetry(wait)
request_wait_tracker = get_request_wait_tracker()
- monitor_started = False
+ job_enqueued = False
+ deferred_lock: LockLease = NO_LOCK
if telemetry_id:
request_wait_tracker.register_request(telemetry_id)
watch_manager = self._get_watch_manager()
@@ -763,11 +810,7 @@ class ResourceService:
and not allow_local_path_resolution
and self._resource_processor is not None
):
- from openviking.service.task_tracker import get_task_tracker
- from openviking.storage.queuefs import QueueManager, get_queue_manager
- from openviking.storage.queuefs.understanding_parse_msg import (
- UnderstandingParseMsg,
- )
+ from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
source_name = kwargs.get("source_name")
source_info = _ResourceSourceInfo(
@@ -820,67 +863,40 @@ class ResourceService:
else:
lock_lease = await _reserve_tree(root_uri)
- task_tracker = get_task_tracker()
- task = None
- try:
- task = await task_tracker.create(
- "add_resource",
- resource_id=root_uri,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- await task_tracker.update_stage(
- task.task_id,
- "queued_external_parse",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
-
- lock_handoff = lock_lease.to_handoff()
- msg = UnderstandingParseMsg(
- task_id=task.task_id,
- telemetry_id=telemetry_id or None,
- path=path,
- root_uri=root_uri,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- role=str(ctx.role),
- actor_peer_id=ctx.actor_peer_id,
- reason=reason,
- instruction=instruction,
- build_index=build_index,
- summarize=summarize,
- strict=bool(kwargs.get("strict", False)),
- ignore_dirs=kwargs.get("ignore_dirs"),
- include=kwargs.get("include"),
- exclude=kwargs.get("exclude"),
- directly_upload_media=bool(kwargs.get("directly_upload_media", True)),
- allow_local_path_resolution=allow_local_path_resolution,
- enforce_public_remote_targets=enforce_public_remote_targets,
- args=normalized_args.processor_kwargs,
- source_name=source_name,
- lock_handoff=lock_handoff.to_dict() if lock_handoff else None,
- )
- qm = get_queue_manager()
- await qm.enqueue(QueueManager.EXTERNAL_PARSE, msg.to_dict())
- await lock_lease.handoff()
- monitor_started = True
- except Exception as exc:
- with contextlib.suppress(Exception):
- await lock_lease.close()
-
- if task is not None:
- with contextlib.suppress(Exception):
- await task_tracker.fail(
- task.task_id,
- str(exc),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
-
- raise
+ lock_handoff = lock_lease.to_handoff()
+ msg = AddResourceMsg(
+ task_id=str(uuid4()),
+ telemetry_id=telemetry_id or None,
+ path=path,
+ root_uri=root_uri,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ role=str(ctx.role),
+ actor_peer_id=ctx.actor_peer_id,
+ reason=reason,
+ instruction=instruction,
+ timeout=timeout,
+ build_index=build_index,
+ summarize=summarize,
+ strict=bool(kwargs.get("strict", False)),
+ ignore_dirs=kwargs.get("ignore_dirs"),
+ include=kwargs.get("include"),
+ exclude=kwargs.get("exclude"),
+ directly_upload_media=bool(kwargs.get("directly_upload_media", True)),
+ preserve_structure=kwargs.get("preserve_structure"),
+ create_parent=bool(kwargs.get("create_parent", False)),
+ allow_local_path_resolution=allow_local_path_resolution,
+ enforce_public_remote_targets=enforce_public_remote_targets,
+ args=self._sanitize_watch_processor_kwargs(normalized_args.processor_kwargs),
+ source_name=source_name,
+ lock_handoff=lock_handoff.to_dict() if lock_handoff else None,
+ skip_watch_management=True,
+ )
+ task = await self._enqueue_add_resource_job(msg, resource_lock=lock_lease)
+ lock_lease = NO_LOCK
+ job_enqueued = True
logger.info(
- "[ResourceService] Enqueued UnderstandingParseMsg task_id=%s root_uri=%s",
+ "[ResourceService] Enqueued AddResourceMsg task_id=%s root_uri=%s",
task.task_id,
root_uri,
)
@@ -917,11 +933,14 @@ class ResourceService:
summarize=summarize,
stage_callback=stage_callback,
allow_local_path_resolution=allow_local_path_resolution,
+ defer_post_processing=not wait,
**kwargs,
)
if result.get("status") == "error":
return result
+ prepared = result.pop("_post_process", None)
+ deferred_lock = result.pop("_resource_lock", NO_LOCK)
if wait:
if stage_callback is not None:
stage_result = stage_callback("processing_queue")
@@ -969,6 +988,47 @@ class ResourceService:
root_uri=result.get("root_uri"),
)
telemetry.set("queue.wait.duration_ms", queue_wait_duration_ms)
+ if not wait:
+ from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
+
+ root_uri = result.get("root_uri", "")
+ if not isinstance(prepared, dict):
+ raise InternalError("Deferred resource processing payload is missing")
+ lock_handoff = deferred_lock.to_handoff()
+ msg = AddResourceMsg(
+ task_id=str(uuid4()),
+ root_uri=root_uri,
+ prepared=prepared,
+ telemetry_id=telemetry_id or None,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ role=str(ctx.role),
+ actor_peer_id=ctx.actor_peer_id,
+ lock_handoff=lock_handoff.to_dict() if lock_handoff else None,
+ reason=reason,
+ instruction=instruction,
+ timeout=timeout,
+ build_index=build_index,
+ summarize=summarize,
+ strict=bool(kwargs.get("strict", False)),
+ ignore_dirs=kwargs.get("ignore_dirs"),
+ include=kwargs.get("include"),
+ exclude=kwargs.get("exclude"),
+ directly_upload_media=bool(kwargs.get("directly_upload_media", True)),
+ preserve_structure=kwargs.get("preserve_structure"),
+ create_parent=bool(kwargs.get("create_parent", False)),
+ allow_local_path_resolution=allow_local_path_resolution,
+ enforce_public_remote_targets=enforce_public_remote_targets,
+ source_name=kwargs.get("source_name"),
+ skip_watch_management=True,
+ )
+ task = await self._enqueue_add_resource_job(
+ msg,
+ resource_lock=deferred_lock,
+ )
+ deferred_lock = NO_LOCK
+ result["task_id"] = task.task_id
+ job_enqueued = True
await self._manage_watch_if_needed(
watch_manager=watch_manager,
skip_watch_management=skip_watch_management,
@@ -992,48 +1052,6 @@ class ResourceService:
source_name=kwargs.get("source_name"),
timeout=timeout,
)
- if not wait:
- from openviking.service.task_tracker import get_task_tracker
-
- task_tracker = get_task_tracker()
- root_uri = result.get("root_uri", "")
- task = await task_tracker.create(
- "add_resource",
- resource_id=root_uri,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- result["task_id"] = task.task_id
- if telemetry_id:
- monitor_started = True
- background = asyncio.create_task(
- self._monitor_resource_queue_then_link_memory(
- task.task_id,
- telemetry_id,
- ctx,
- root_uri=root_uri,
- reason=reason,
- source_name=kwargs.get("source_name"),
- timeout=timeout,
- )
- )
- self._background_tasks.add(background)
- background.add_done_callback(self._background_tasks.discard)
- else:
- monitor_started = True
- background = asyncio.create_task(
- self._monitor_resource_queue_then_link_memory(
- task.task_id,
- None,
- ctx,
- root_uri=root_uri,
- reason=reason,
- source_name=kwargs.get("source_name"),
- timeout=timeout,
- )
- )
- self._background_tasks.add(background)
- background.add_done_callback(self._background_tasks.discard)
return result
except Exception as exc:
telemetry.set_error(
@@ -1047,9 +1065,11 @@ class ResourceService:
"resource.request.duration_ms",
round((time.perf_counter() - request_start) * 1000, 3),
)
- if wait or not telemetry_id or not monitor_started:
+ if wait or not telemetry_id or not job_enqueued:
get_request_wait_tracker().cleanup(telemetry_id)
unregister_wait_telemetry(telemetry_id)
+ if deferred_lock.active:
+ await deferred_lock.close()
async def _link_resource_reason_memory(
self,
@@ -1080,66 +1100,6 @@ class ResourceService:
logger.warning("[ResourceService] Failed to link resource reason memory: %s", exc)
result.setdefault("warnings", []).append(f"Memory linking failed: {exc}")
- async def _monitor_resource_queue_then_link_memory(
- self,
- task_id: str,
- telemetry_id: Optional[str],
- ctx: RequestContext,
- *,
- root_uri: str,
- reason: str,
- source_name: Optional[str],
- timeout: Optional[float],
- ) -> None:
- from openviking.service.task_tracker import get_task_tracker
-
- task_tracker = get_task_tracker()
- request_wait_tracker = get_request_wait_tracker()
- await task_tracker.start(task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)
- try:
- if telemetry_id:
- await request_wait_tracker.wait_for_request(telemetry_id)
- status = request_wait_tracker.build_queue_status(telemetry_id)
- else:
- status = build_queue_status_payload(
- await get_queue_manager().wait_complete(timeout=timeout)
- )
- errors = sum(int(group.get("error_count", 0) or 0) for group in status.values())
- if errors:
- await task_tracker.fail(
- task_id,
- f"queue processing failed: {status}",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- return
-
- result: Dict[str, Any] = {"root_uri": root_uri, "queue_status": status}
- await self._link_resource_reason_memory(
- result=result,
- ctx=ctx,
- reason=reason,
- source_name=source_name,
- timeout=timeout,
- )
- await task_tracker.complete(
- task_id,
- result,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- except Exception as exc:
- await task_tracker.fail(
- task_id,
- str(exc),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- finally:
- if telemetry_id:
- request_wait_tracker.cleanup(telemetry_id)
- unregister_wait_telemetry(telemetry_id)
-
async def _monitor_queue_processing(
self,
task_id: str,
@@ -1297,7 +1257,9 @@ class ResourceService:
"wait=true (Connector imports run asynchronously; poll the returned task_id)"
)
if reason:
- unsupported.append("reason (Connector imports cannot preserve resource-reason semantics)")
+ unsupported.append(
+ "reason (Connector imports cannot preserve resource-reason semantics)"
+ )
if instruction:
unsupported.append("instruction")
if not build_index:
diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.py
index c020497b4..40e5b0a9f 100644
--- a/openviking/service/task_tracker.py
+++ b/openviking/service/task_tracker.py
@@ -39,6 +39,9 @@ class TaskStatus(str, Enum):
FAILED = "failed"
+_TERMINAL_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED)
+
+
@dataclass
class TaskRecord:
"""Immutable snapshot of an async task."""
@@ -333,7 +336,7 @@ class TaskTracker:
"""Transition task to RUNNING."""
async with self._async_lock:
task = await self._load_for_update(task_id, account_id, user_id)
- if task:
+ if task and task.status not in _TERMINAL_STATUSES:
task.status = TaskStatus.RUNNING
if stage is not None:
task.stage = stage
@@ -352,7 +355,7 @@ class TaskTracker:
"""Update task stage without changing its lifecycle status."""
async with self._async_lock:
task = await self._load_for_update(task_id, account_id, user_id)
- if task:
+ if task and task.status not in _TERMINAL_STATUSES:
task.stage = stage
task.updated_at = time.time()
await self._store.update(task)
@@ -367,9 +370,10 @@ class TaskTracker:
user_id: Optional[str] = None,
) -> None:
"""Transition task to COMPLETED with optional result."""
+ transitioned = False
async with self._async_lock:
task = await self._load_for_update(task_id, account_id, user_id)
- if task:
+ if task and task.status not in _TERMINAL_STATUSES:
task.status = TaskStatus.COMPLETED
task.stage = "completed"
task.result = result
@@ -377,7 +381,9 @@ class TaskTracker:
await self._store.update(task)
with self._lock:
self._tasks[task.task_id] = task
- logger.info("[TaskTracker] Task %s completed", task_id)
+ transitioned = True
+ if transitioned:
+ logger.info("[TaskTracker] Task %s completed", task_id)
async def fail(
self,
@@ -387,9 +393,10 @@ class TaskTracker:
user_id: Optional[str] = None,
) -> None:
"""Transition task to FAILED with sanitized error."""
+ transitioned = False
async with self._async_lock:
task = await self._load_for_update(task_id, account_id, user_id)
- if task:
+ if task and task.status not in _TERMINAL_STATUSES:
task.status = TaskStatus.FAILED
task.stage = "failed"
task.error = _sanitize_error(error)
@@ -397,7 +404,9 @@ class TaskTracker:
await self._store.update(task)
with self._lock:
self._tasks[task.task_id] = task
- logger.warning("[TaskTracker] Task %s failed: %s", task_id, _sanitize_error(error))
+ transitioned = True
+ if transitioned:
+ logger.warning("[TaskTracker] Task %s failed: %s", task_id, _sanitize_error(error))
async def get(
self,
diff --git a/openviking/storage/queuefs/add_resource_msg.py b/openviking/storage/queuefs/add_resource_msg.py
new file mode 100644
index 000000000..16dd5c624
--- /dev/null
+++ b/openviking/storage/queuefs/add_resource_msg.py
@@ -0,0 +1,111 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Persistent add-resource queue message (legacy ExternalParse queue payload)."""
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, Optional
+
+
+@dataclass(kw_only=True)
+class AddResourceMsg:
+ task_id: str
+ root_uri: str
+ account_id: str
+ user_id: str
+ role: str
+ path: str = ""
+ telemetry_id: Optional[str] = None
+ prepared: Optional[Dict[str, Any]] = None
+ lock_handoff: Optional[Dict[str, Any]] = None
+ actor_peer_id: Optional[str] = None
+ reason: str = ""
+ instruction: str = ""
+ timeout: Optional[float] = None
+ build_index: bool = True
+ summarize: bool = False
+ strict: bool = False
+ ignore_dirs: Optional[str] = None
+ include: Optional[str] = None
+ exclude: Optional[str] = None
+ directly_upload_media: bool = True
+ preserve_structure: Optional[bool] = None
+ create_parent: bool = False
+ allow_local_path_resolution: bool = True
+ enforce_public_remote_targets: bool = False
+ args: Dict[str, Any] = field(default_factory=dict)
+ lock_handoff_retry: int = 0
+ source_name: Optional[str] = None
+ watch_interval: float = 0
+ skip_watch_management: bool = True
+
+ def to_dict(self) -> Dict[str, Any]:
+ data = asdict(self)
+ if self.prepared is not None:
+ data["args"] = {}
+ return data
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> "AddResourceMsg":
+ if not isinstance(data, dict) or not data:
+ raise ValueError("Data dictionary is empty")
+ task_id = data.get("task_id")
+ path = data.get("path")
+ root_uri = data.get("root_uri")
+ prepared = data.get("prepared") if isinstance(data.get("prepared"), dict) else None
+ args = dict(data.get("args", {})) if isinstance(data.get("args"), dict) else {}
+ legacy_retry = args.pop("_lock_handoff_retry", 0)
+ try:
+ lock_handoff_retry = max(0, int(data.get("lock_handoff_retry", legacy_retry) or 0))
+ except (TypeError, ValueError):
+ lock_handoff_retry = 0
+ if prepared is not None:
+ args.clear()
+ if not task_id or (not path and not prepared) or not root_uri:
+ missing = []
+ if not task_id:
+ missing.append("task_id")
+ if not path and not prepared:
+ missing.append("path or prepared")
+ if not root_uri:
+ missing.append("root_uri")
+ raise ValueError(f"Missing required fields: {missing}")
+
+ return cls(
+ task_id=str(task_id),
+ path=str(path or ""),
+ root_uri=str(root_uri),
+ account_id=str(data.get("account_id", "default")),
+ user_id=str(data.get("user_id", "default")),
+ role=str(data.get("role", "root")),
+ actor_peer_id=data.get("actor_peer_id"),
+ telemetry_id=str(data.get("telemetry_id"))
+ if isinstance(data.get("telemetry_id"), str)
+ else None,
+ lock_handoff=data.get("lock_handoff")
+ if isinstance(data.get("lock_handoff"), dict)
+ else None,
+ reason=str(data.get("reason", "")),
+ instruction=str(data.get("instruction", "")),
+ timeout=float(data["timeout"]) if data.get("timeout") is not None else None,
+ build_index=bool(data.get("build_index", True)),
+ summarize=bool(data.get("summarize", False)),
+ strict=bool(data.get("strict", False)),
+ ignore_dirs=data.get("ignore_dirs"),
+ include=data.get("include"),
+ exclude=data.get("exclude"),
+ directly_upload_media=bool(data.get("directly_upload_media", True)),
+ preserve_structure=(
+ bool(data["preserve_structure"])
+ if data.get("preserve_structure") is not None
+ else None
+ ),
+ create_parent=bool(data.get("create_parent", False)),
+ allow_local_path_resolution=bool(data.get("allow_local_path_resolution", True)),
+ enforce_public_remote_targets=bool(data.get("enforce_public_remote_targets", False)),
+ args=args,
+ lock_handoff_retry=lock_handoff_retry,
+ source_name=data.get("source_name"),
+ prepared=prepared,
+ watch_interval=float(data.get("watch_interval", 0) or 0),
+ skip_watch_management=bool(data.get("skip_watch_management", True)),
+ )
diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.py
new file mode 100644
index 000000000..d5e3ec6e5
--- /dev/null
+++ b/openviking/storage/queuefs/add_resource_processor.py
@@ -0,0 +1,204 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Durable add-resource consumer on the legacy ExternalParse queue."""
+
+import asyncio
+import concurrent.futures
+import json
+from contextlib import suppress
+from typing import Any, Dict, Optional
+
+from openviking.observability.context import bind_execution_context
+from openviking.server.identity import RequestContext, Role
+from openviking.service.task_tracker import TaskStatus, get_task_tracker
+from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
+from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.telemetry import bind_telemetry, resolve_telemetry
+from openviking.telemetry.resource_summary import summarize_queue_errors
+from openviking_cli.session.user_id import UserIdentifier
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class AddResourceProcessor(DequeueHandlerBase):
+ """Own an add-resource task until it reaches a terminal state and can be ACKed."""
+
+ def __init__(
+ self,
+ resource_service: Any,
+ service_loop: asyncio.AbstractEventLoop,
+ ):
+ self._resource_service = resource_service
+ self._service_loop = service_loop
+
+ async def _load_lock(self, msg: AddResourceMsg, ctx: RequestContext) -> Any:
+ if msg.lock_handoff is None:
+ return None
+ from openviking.storage.transaction.lock_lease import LockHandoffRef, OwnedLockLease
+
+ ref = LockHandoffRef.from_value(msg.lock_handoff)
+ if ref is None:
+ raise ValueError("Invalid lock_handoff")
+ try:
+ return await OwnedLockLease.from_handoff(ref)
+ except Exception as handoff_error:
+ try:
+ return await self._resource_service.reacquire_add_resource_job_lock(
+ msg.root_uri,
+ ctx,
+ )
+ except Exception:
+ raise handoff_error
+
+ async def _requeue_lock_handoff(self, msg: AddResourceMsg, exc: Exception) -> bool:
+ if msg.lock_handoff_retry >= 2:
+ return False
+
+ from openviking.storage.queuefs import QueueManager, get_queue_manager
+
+ payload = msg.to_dict()
+ payload["lock_handoff_retry"] = msg.lock_handoff_retry + 1
+ await get_queue_manager().enqueue(QueueManager.ADD_RESOURCE, payload)
+ logger.warning(
+ "[AddResource] Requeued task %s after lock handoff failure: %s",
+ msg.task_id,
+ exc,
+ )
+ self.report_requeue()
+ self.report_success()
+ return True
+
+ async def _process(self, msg: AddResourceMsg, data: Dict[str, Any]) -> None:
+ ctx = RequestContext(
+ user=UserIdentifier(msg.account_id, msg.user_id),
+ role=Role(msg.role),
+ actor_peer_id=msg.actor_peer_id,
+ )
+ tracker = get_task_tracker()
+ task = await tracker.create(
+ "add_resource",
+ resource_id=msg.root_uri,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ task_id=msg.task_id,
+ )
+ if task.status in (TaskStatus.COMPLETED, TaskStatus.FAILED):
+ self.report_success()
+ return None
+
+ resource_lock = None
+ try:
+ resource_lock = await self._load_lock(msg, ctx)
+ except Exception as exc:
+ if await self._requeue_lock_handoff(msg, exc):
+ return None
+ await tracker.fail(
+ msg.task_id,
+ f"Invalid lock_handoff: {exc}",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error(f"Invalid lock_handoff: {exc}", data)
+ return None
+
+ telemetry_id = msg.telemetry_id or ""
+ telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None
+ if telemetry is None:
+ from openviking.telemetry.operation import OperationTelemetry
+
+ telemetry = OperationTelemetry(operation="add_resource_job", enabled=False)
+ if telemetry_id:
+ telemetry.telemetry_id = telemetry_id
+
+ async def _set_stage(stage: str) -> None:
+ await tracker.update_stage(
+ msg.task_id,
+ stage,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+
+ with bind_execution_context(), bind_telemetry(telemetry):
+ try:
+ await tracker.start(
+ msg.task_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ stage="queued",
+ )
+ result = await self._resource_service.execute_add_resource_job(
+ msg,
+ ctx=ctx,
+ resource_lock=resource_lock,
+ stage_callback=_set_stage,
+ )
+ if result.get("status") == "error":
+ errors = result.get("errors") or ["resource processing failed"]
+ await tracker.fail(
+ msg.task_id,
+ "; ".join(str(error) for error in errors),
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error("resource processing failed", data)
+ return None
+ queue_errors = summarize_queue_errors(result.get("queue_status"))
+ if queue_errors:
+ await tracker.fail(
+ msg.task_id,
+ "queue processing failed: " + "; ".join(queue_errors),
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error("queue processing failed", data)
+ return None
+ await tracker.complete(
+ msg.task_id,
+ result,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_success()
+ return None
+ except asyncio.CancelledError:
+ # Leave both task and QueueFS message active; RecoverStale owns restart recovery.
+ raise
+ except Exception as exc:
+ await tracker.fail(
+ msg.task_id,
+ str(exc),
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error(str(exc), data)
+ return None
+ finally:
+ with suppress(Exception):
+ if resource_lock is not None:
+ await resource_lock.close()
+
+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ if not data:
+ return None
+ try:
+ if not isinstance(data, dict):
+ raise ValueError("Queue message must be an object")
+ payload = data.get("data", data)
+ if isinstance(payload, str):
+ payload = json.loads(payload)
+ msg = AddResourceMsg.from_dict(payload)
+ except (json.JSONDecodeError, TypeError, ValueError) as exc:
+ self.report_error(str(exc), data)
+ return None
+
+ future: concurrent.futures.Future[None] = asyncio.run_coroutine_threadsafe(
+ self._process(msg, data),
+ self._service_loop,
+ )
+ try:
+ await asyncio.wrap_future(future)
+ except asyncio.CancelledError:
+ future.cancel()
+ raise
+ return None
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index 1cfd6d4c9..3de4efc62 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -69,7 +69,8 @@ class QueueManager:
# Standard queue names
EMBEDDING = "Embedding"
SEMANTIC = "Semantic"
- EXTERNAL_PARSE = "ExternalParse"
+ # Keep the on-disk queue name stable so pre-upgrade jobs remain recoverable.
+ ADD_RESOURCE = "ExternalParse"
SESSION_COMMIT = "SessionCommit"
def __init__(
@@ -157,7 +158,7 @@ class QueueManager:
if queue.name == self.EMBEDDING:
max_concurrent = self._max_concurrent_embedding
- elif queue.name in {self.EXTERNAL_PARSE, self.SESSION_COMMIT}:
+ elif queue.name in {self.ADD_RESOURCE, self.SESSION_COMMIT}:
max_concurrent = self._max_concurrent_external_parse
else:
max_concurrent = self._max_concurrent_semantic
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index e26c51d91..016a03e15 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -748,19 +748,19 @@ class SemanticProcessor(DequeueHandlerBase):
lock: LockLease = NO_LOCK,
) -> DiffResult:
viking_fs = get_viking_fs()
+ if not await viking_fs.exists(root_uri, ctx=ctx):
+ raise FileNotFoundError(
+ f"Semantic source no longer exists; refusing to sync into {target_uri}: {root_uri}"
+ )
diff = DiffResult()
lock_handle = lock.handle
async def list_children(dir_uri: str) -> Tuple[Dict[str, str], Dict[str, str]]:
files: Dict[str, str] = {}
dirs: Dict[str, str] = {}
- try:
- entries = await viking_fs.ls(
- dir_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx
- )
- except Exception as e:
- logger.error(f"[SyncDiff] Failed to list {dir_uri}: {e}")
- return files, dirs
+ entries = await viking_fs.ls(
+ dir_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx
+ )
for entry in entries:
name = entry.get("name", "")
diff --git a/openviking/storage/queuefs/understanding_parse_msg.py b/openviking/storage/queuefs/understanding_parse_msg.py
deleted file mode 100644
index 12ea5c731..000000000
--- a/openviking/storage/queuefs/understanding_parse_msg.py
+++ /dev/null
@@ -1,155 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""UnderstandingParseMsg: External parse queue message for UnderstandingAPI."""
-
-import json
-from dataclasses import asdict, dataclass, field
-from datetime import datetime
-from typing import Any, Dict, Optional
-from uuid import uuid4
-
-
-@dataclass
-class UnderstandingParseMsg:
- id: str
- task_id: str
- telemetry_id: Optional[str]
- path: str
- root_uri: str
- lock_handoff: Optional[Dict[str, Any]] = None
- status: str = "pending"
- timestamp: int = int(datetime.now().timestamp())
- account_id: str = "default"
- user_id: str = "default"
- role: str = "root"
- actor_peer_id: Optional[str] = None
- reason: str = ""
- instruction: str = ""
- build_index: bool = True
- summarize: bool = False
- strict: bool = False
- ignore_dirs: Optional[str] = None
- include: Optional[str] = None
- exclude: Optional[str] = None
- directly_upload_media: bool = True
- allow_local_path_resolution: bool = True
- enforce_public_remote_targets: bool = False
- args: Dict[str, Any] = field(default_factory=dict)
- source_name: Optional[str] = None
-
- def __init__(
- self,
- *,
- task_id: str,
- path: str,
- root_uri: str,
- account_id: str,
- user_id: str,
- role: str,
- actor_peer_id: Optional[str] = None,
- telemetry_id: Optional[str] = None,
- lock_handoff: Optional[Dict[str, Any]] = None,
- reason: str = "",
- instruction: str = "",
- build_index: bool = True,
- summarize: bool = False,
- strict: bool = False,
- ignore_dirs: Optional[str] = None,
- include: Optional[str] = None,
- exclude: Optional[str] = None,
- directly_upload_media: bool = True,
- allow_local_path_resolution: bool = True,
- enforce_public_remote_targets: bool = False,
- args: Optional[Dict[str, Any]] = None,
- source_name: Optional[str] = None,
- ):
- self.id = str(uuid4())
- self.task_id = task_id
- self.telemetry_id = telemetry_id
- self.path = path
- self.root_uri = root_uri
- self.account_id = account_id
- self.user_id = user_id
- self.role = role
- self.actor_peer_id = actor_peer_id
- self.lock_handoff = lock_handoff
- self.reason = reason
- self.instruction = instruction
- self.build_index = build_index
- self.summarize = summarize
- self.strict = strict
- self.ignore_dirs = ignore_dirs
- self.include = include
- self.exclude = exclude
- self.directly_upload_media = directly_upload_media
- self.allow_local_path_resolution = allow_local_path_resolution
- self.enforce_public_remote_targets = enforce_public_remote_targets
- self.args = args or {}
- self.source_name = source_name
-
- def to_dict(self) -> Dict[str, Any]:
- return asdict(self)
-
- def to_json(self) -> str:
- return json.dumps(self.to_dict())
-
- @classmethod
- def from_dict(cls, data: Dict[str, Any]) -> "UnderstandingParseMsg":
- if not data:
- raise ValueError("Data dictionary is empty")
- task_id = data.get("task_id")
- path = data.get("path")
- root_uri = data.get("root_uri")
- if not task_id or not path or not root_uri:
- missing = []
- if not task_id:
- missing.append("task_id")
- if not path:
- missing.append("path")
- if not root_uri:
- missing.append("root_uri")
- raise ValueError(f"Missing required fields: {missing}")
-
- obj = cls(
- task_id=str(task_id),
- path=str(path),
- root_uri=str(root_uri),
- account_id=str(data.get("account_id", "default")),
- user_id=str(data.get("user_id", "default")),
- role=str(data.get("role", "root")),
- actor_peer_id=data.get("actor_peer_id"),
- telemetry_id=str(data.get("telemetry_id"))
- if isinstance(data.get("telemetry_id"), str)
- else None,
- lock_handoff=data.get("lock_handoff")
- if isinstance(data.get("lock_handoff"), dict)
- else None,
- reason=str(data.get("reason", "")),
- instruction=str(data.get("instruction", "")),
- build_index=bool(data.get("build_index", True)),
- summarize=bool(data.get("summarize", False)),
- strict=bool(data.get("strict", False)),
- ignore_dirs=data.get("ignore_dirs"),
- include=data.get("include"),
- exclude=data.get("exclude"),
- directly_upload_media=bool(data.get("directly_upload_media", True)),
- allow_local_path_resolution=bool(data.get("allow_local_path_resolution", True)),
- enforce_public_remote_targets=bool(data.get("enforce_public_remote_targets", False)),
- args=data.get("args") if isinstance(data.get("args"), dict) else {},
- source_name=data.get("source_name"),
- )
- if data.get("id"):
- obj.id = str(data["id"])
- if "status" in data:
- obj.status = str(data["status"])
- if "timestamp" in data:
- obj.timestamp = int(data["timestamp"])
- return obj
-
- @classmethod
- def from_json(cls, json_str: str) -> "UnderstandingParseMsg":
- try:
- data = json.loads(json_str)
- except json.JSONDecodeError as e:
- raise ValueError(f"Invalid JSON string: {e}")
- return cls.from_dict(data)
diff --git a/openviking/storage/queuefs/understanding_parse_processor.py b/openviking/storage/queuefs/understanding_parse_processor.py
deleted file mode 100644
index 38f6e770b..000000000
--- a/openviking/storage/queuefs/understanding_parse_processor.py
+++ /dev/null
@@ -1,344 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""UnderstandingParseProcessor: Processes UnderstandingParseMsg messages."""
-
-import asyncio
-import json
-
-from contextlib import suppress
-from typing import Any, Dict, Optional
-
-from openviking.observability.context import (
- bind_execution_context,
-)
-from openviking.server.identity import RequestContext, Role
-from openviking.service.task_tracker import get_task_tracker
-from openviking.storage.queuefs.named_queue import DequeueHandlerBase
-from openviking.storage.queuefs.understanding_parse_msg import UnderstandingParseMsg
-from openviking.telemetry import bind_telemetry, bind_telemetry_stage, resolve_telemetry
-from openviking_cli.session.user_id import UserIdentifier
-from openviking_cli.utils.logger import get_logger
-
-logger = get_logger(__name__)
-
-
-class UnderstandingParseProcessor(DequeueHandlerBase):
- def __init__(self, resource_processor: Any, resource_memory_link_service: Any = None):
- self._resource_processor = resource_processor
- self._resource_memory_link_service = resource_memory_link_service
- self._background_tasks: set[asyncio.Task] = set()
-
- async def _monitor_queue_then_link_memory(
- self,
- *,
- task_id: str,
- telemetry_id: str,
- ctx: RequestContext,
- result: Dict[str, Any],
- reason: str,
- source_name: Optional[str],
- ) -> None:
- from openviking.telemetry.resource_summary import unregister_wait_telemetry
- from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
-
- task_tracker = get_task_tracker()
- request_wait_tracker = get_request_wait_tracker()
- try:
- request_wait_tracker.register_request(telemetry_id)
- await request_wait_tracker.wait_for_request(telemetry_id)
- status = request_wait_tracker.build_queue_status(telemetry_id)
- errors = sum(int(group.get("error_count", 0) or 0) for group in status.values())
- if errors:
- await task_tracker.fail(
- task_id,
- f"queue processing failed: {status}",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- self.report_error("queue processing failed", {"task_id": task_id})
- return
-
- result["queue_status"] = status
- if self._resource_memory_link_service and (reason or "").strip():
- root_uri = result.get("root_uri")
- if root_uri:
- try:
- link_result = await self._resource_memory_link_service.on_resource_added(
- ctx=ctx,
- resource_uri=root_uri,
- reason=reason,
- source_name=source_name,
- timeout=None,
- )
- result["memory_linking"] = link_result
- except Exception as exc:
- logger.warning(
- "[UnderstandingParse] Failed to link resource reason memory: %s",
- exc,
- )
- result.setdefault("warnings", []).append(f"Memory linking failed: {exc}")
-
- await task_tracker.complete(
- task_id,
- result,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- self.report_success()
- except Exception as exc:
- await task_tracker.fail(
- task_id,
- str(exc),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- self.report_error(str(exc), {"task_id": task_id})
- finally:
- request_wait_tracker.cleanup(telemetry_id)
- unregister_wait_telemetry(telemetry_id)
-
- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
- if not data:
- return None
-
- payload: Dict[str, Any] = data
- if isinstance(data, dict) and "data" in data:
- inner = data.get("data")
- if isinstance(inner, str) and inner.strip():
- try:
- decoded = json.loads(inner)
- if isinstance(decoded, dict):
- payload = decoded
- except Exception:
- payload = data
- elif isinstance(inner, dict):
- payload = inner
- if isinstance(payload, dict) and data.get("id") and not payload.get("id"):
- payload["id"] = data["id"]
-
- msg = UnderstandingParseMsg.from_dict(payload)
- ctx = RequestContext(
- user=UserIdentifier(msg.account_id, msg.user_id),
- role=Role(msg.role),
- actor_peer_id=msg.actor_peer_id,
- )
-
- task_tracker = get_task_tracker()
- lock_lease = None
- if msg.lock_handoff is not None:
- try:
- from openviking.storage.transaction.lock_lease import (
- LockHandoffRef,
- OwnedLockLease,
- )
-
- ref = LockHandoffRef.from_value(msg.lock_handoff)
- if ref is None:
- raise ValueError("Invalid lock_handoff")
- lock_lease = await OwnedLockLease.from_handoff(ref)
- except Exception as exc:
- retry_key = "_lock_handoff_retry"
- raw_retry = (msg.args or {}).get(retry_key, 0)
- try:
- retry_count = int(raw_retry or 0)
- except (TypeError, ValueError):
- retry_count = 0
-
- max_retries = 2
- if retry_count < max_retries:
- from openviking.storage.queuefs import QueueManager, get_queue_manager
-
- qm = get_queue_manager()
- if qm is None:
- raise
- new_args = dict(msg.args or {})
- new_args[retry_key] = retry_count + 1
- retry_msg = UnderstandingParseMsg(
- task_id=msg.task_id,
- telemetry_id=msg.telemetry_id,
- path=msg.path,
- root_uri=msg.root_uri,
- account_id=msg.account_id,
- user_id=msg.user_id,
- role=msg.role,
- actor_peer_id=msg.actor_peer_id,
- lock_handoff=msg.lock_handoff,
- reason=msg.reason,
- instruction=msg.instruction,
- build_index=msg.build_index,
- summarize=msg.summarize,
- strict=msg.strict,
- ignore_dirs=msg.ignore_dirs,
- include=msg.include,
- exclude=msg.exclude,
- directly_upload_media=msg.directly_upload_media,
- allow_local_path_resolution=msg.allow_local_path_resolution,
- enforce_public_remote_targets=msg.enforce_public_remote_targets,
- args=new_args,
- source_name=msg.source_name,
- )
- await qm.enqueue(QueueManager.EXTERNAL_PARSE, retry_msg.to_dict())
- self.report_requeue()
- self.report_success()
- return None
-
- await task_tracker.fail(
- msg.task_id,
- f"Invalid lock_handoff: {exc}",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- self.report_error(f"Invalid lock_handoff: {exc}", data)
- return None
-
- logger.info(
- "[UnderstandingParse] Dequeued task_id=%s root_uri=%s",
- msg.task_id,
- msg.root_uri,
- )
- resource_processor = self._resource_processor
- if resource_processor is None:
- raise RuntimeError("ResourceProcessor is None")
-
- telemetry_id = msg.telemetry_id or ""
- if telemetry_id:
- from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
-
- get_request_wait_tracker().register_request(telemetry_id)
-
- telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None
- if telemetry_id and telemetry is None:
- from openviking.telemetry.operation import OperationTelemetry
-
- telemetry = OperationTelemetry(operation="noop", enabled=False)
- telemetry.telemetry_id = telemetry_id
-
- with bind_execution_context(), (bind_telemetry(telemetry) if telemetry else suppress()):
- try:
- await task_tracker.start(
- msg.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id
- )
- await task_tracker.update_stage(
- msg.task_id,
- "parsing",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
-
- bind_telemetry_stage("parse_understanding")
- kwargs: Dict[str, Any] = {}
- if lock_lease is not None and getattr(lock_lease, "active", False):
- kwargs["resource_lock"] = lock_lease
- if msg.enforce_public_remote_targets:
- from openviking.utils.network_guard import ensure_public_remote_target
-
- kwargs.setdefault("request_validator", ensure_public_remote_target)
- result = await resource_processor.process_resource(
- path=msg.path,
- ctx=ctx,
- reason=msg.reason,
- instruction=msg.instruction,
- scope="resources",
- to=msg.root_uri,
- parent=None,
- build_index=msg.build_index,
- summarize=msg.summarize,
- allow_local_path_resolution=msg.allow_local_path_resolution,
- enforce_public_remote_targets=msg.enforce_public_remote_targets,
- strict=msg.strict,
- source_name=msg.source_name,
- ignore_dirs=msg.ignore_dirs,
- include=msg.include,
- exclude=msg.exclude,
- directly_upload_media=msg.directly_upload_media,
- args=msg.args,
- skip_watch_management=True,
- watch_interval=0,
- **kwargs,
- )
-
- if result.get("status") == "error":
- errors = result.get("errors") or ["resource processing failed"]
- await task_tracker.fail(
- msg.task_id,
- "; ".join(str(error) for error in errors),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- self.report_error("resource processing failed", data)
- return None
-
- if telemetry_id:
- await task_tracker.update_stage(
- msg.task_id,
- "processing_queue",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- monitor = asyncio.create_task(
- self._monitor_queue_then_link_memory(
- task_id=msg.task_id,
- telemetry_id=telemetry_id,
- ctx=ctx,
- result=result,
- reason=msg.reason,
- source_name=msg.source_name,
- )
- )
- self._background_tasks.add(monitor)
- monitor.add_done_callback(self._background_tasks.discard)
- return None
-
- if self._resource_memory_link_service and (msg.reason or "").strip():
- root_uri = result.get("root_uri")
- if root_uri:
- try:
- link_result = await self._resource_memory_link_service.on_resource_added(
- ctx=ctx,
- resource_uri=root_uri,
- reason=msg.reason,
- source_name=msg.source_name,
- timeout=None,
- )
- result["memory_linking"] = link_result
- except Exception as exc:
- logger.warning(
- "[UnderstandingParse] Failed to link resource reason memory: %s",
- exc,
- )
- result.setdefault("warnings", []).append(
- f"Memory linking failed: {exc}"
- )
-
- await task_tracker.complete(
- msg.task_id,
- result,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- self.report_success()
- return None
- except asyncio.CancelledError:
- await task_tracker.fail(
- msg.task_id,
- "external parse cancelled",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- self.report_error("external parse cancelled", data)
- raise
- except Exception as exc:
- await task_tracker.fail(
- msg.task_id,
- str(exc),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- self.report_error(str(exc), data)
- return None
- finally:
- try:
- if lock_lease is not None:
- await lock_lease.close()
- except Exception:
- pass
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index 7731a27e7..74c64f04f 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -135,6 +135,7 @@ class ResourceProcessor:
"errors": [],
"source_path": None,
}
+ defer_post_processing = bool(kwargs.pop("defer_post_processing", False))
preacquired_lock = kwargs.pop("resource_lock", NO_LOCK) or NO_LOCK
telemetry = get_current_telemetry()
@@ -360,56 +361,27 @@ class ResourceProcessor:
except Exception:
pass
- # ============ Phase 4: Optional Steps ============
- build_index = kwargs.get("build_index", True)
- temp_uri_for_summarize = temp_uri or parse_result.temp_dir_path
- should_summarize = summarize or build_index
- if should_summarize:
- skip_vec = not build_index
- is_code_repo = parse_result.source_format == "repository"
- try:
- stage_start = time.perf_counter()
- stage_status = "ok"
- with telemetry.measure("resource.summarize"):
- summary_result = await self._get_summarizer().summarize(
- resource_uris=[result["root_uri"]],
- ctx=ctx,
- skip_vectorization=skip_vec,
- lock=resource_lock,
- temp_uris=[temp_uri_for_summarize],
- is_code_repo=is_code_repo,
- target_preexisting=target_preexisting,
- **kwargs,
- )
- if (
- resource_lock.active
- and summary_result.get("status") == "success"
- and summary_result.get("enqueued_count", 0) > 0
- ):
- await resource_lock.handoff()
- resource_lock = NO_LOCK
- except Exception as e:
- logger.error(f"Summarization failed: {e}")
- result["warnings"] = result.get("warnings", []) + [f"Summarization failed: {e}"]
- stage_status = "error"
- finally:
- try:
- ResourceIngestionEventDataSource.record_stage(
- stage="summarize",
- status=str(stage_status),
- duration_seconds=float(time.perf_counter() - stage_start),
- account_id=getattr(ctx, "account_id", None),
- )
- except Exception:
- pass
-
- if resource_lock.active:
- if not should_summarize and temp_uri and not source_committed:
- viking_fs = get_viking_fs()
- await viking_fs.persist_temp_tree(temp_uri, root_uri, ctx=ctx)
- await rewrite_image_uris(root_uri, ctx=ctx, lock_handle=resource_lock.handle)
- await viking_fs.delete_temp(parse_result.temp_dir_path, ctx=ctx)
- await resource_lock.close()
+ prepared = {
+ "root_uri": root_uri,
+ "temp_uri": temp_uri or parse_result.temp_dir_path,
+ "temp_dir_path": parse_result.temp_dir_path,
+ "source_committed": source_committed,
+ "target_preexisting": target_preexisting,
+ "is_code_repo": parse_result.source_format == "repository",
+ }
+ if defer_post_processing:
+ result["_post_process"] = prepared
+ result["_resource_lock"] = resource_lock
+ else:
+ post_result = await self.finish_prepared_resource(
+ prepared,
+ ctx=ctx,
+ resource_lock=resource_lock,
+ summarize=summarize,
+ **kwargs,
+ )
+ if post_result.get("warnings"):
+ result.setdefault("warnings", []).extend(post_result["warnings"])
# 恢复原始 temp_uri 用于输出
if original_temp_uri is not None:
@@ -417,6 +389,80 @@ class ResourceProcessor:
return result
+ async def finish_prepared_resource(
+ self,
+ prepared: Dict[str, Any],
+ *,
+ ctx: RequestContext,
+ resource_lock: LockLease = NO_LOCK,
+ summarize: bool = False,
+ **kwargs: Any,
+ ) -> Dict[str, Any]:
+ """Run the queue-producing phase for a resource already stored in VikingFS."""
+ from openviking.metrics.datasources.resource import ResourceIngestionEventDataSource
+
+ root_uri = str(prepared.get("root_uri") or "")
+ temp_uri = prepared.get("temp_uri")
+ temp_dir_path = prepared.get("temp_dir_path")
+ source_committed = bool(prepared.get("source_committed"))
+ target_preexisting = bool(prepared.get("target_preexisting"))
+ build_index = bool(kwargs.get("build_index", True))
+ should_summarize = summarize or build_index
+ result: Dict[str, Any] = {"status": "success", "root_uri": root_uri}
+
+ if should_summarize:
+ stage_start = time.perf_counter()
+ stage_status = "ok"
+ try:
+ with get_current_telemetry().measure("resource.summarize"):
+ summary_result = await self._get_summarizer().summarize(
+ resource_uris=[root_uri],
+ ctx=ctx,
+ skip_vectorization=not build_index,
+ lock=resource_lock,
+ temp_uris=[temp_uri],
+ is_code_repo=bool(prepared.get("is_code_repo")),
+ target_preexisting=target_preexisting,
+ **kwargs,
+ )
+ if (
+ resource_lock.active
+ and summary_result.get("status") == "success"
+ and summary_result.get("enqueued_count", 0) > 0
+ ):
+ await resource_lock.handoff()
+ resource_lock = NO_LOCK
+ except Exception as exc:
+ logger.error("Summarization failed: %s", exc)
+ result["warnings"] = [f"Summarization failed: {exc}"]
+ stage_status = "error"
+ finally:
+ try:
+ ResourceIngestionEventDataSource.record_stage(
+ stage="summarize",
+ status=stage_status,
+ duration_seconds=float(time.perf_counter() - stage_start),
+ account_id=getattr(ctx, "account_id", None),
+ )
+ except Exception:
+ pass
+
+ if resource_lock.active:
+ try:
+ if not should_summarize and temp_uri and not source_committed:
+ viking_fs = get_viking_fs()
+ await viking_fs.persist_temp_tree(temp_uri, root_uri, ctx=ctx)
+ await rewrite_image_uris(
+ root_uri,
+ ctx=ctx,
+ lock_handle=resource_lock.handle,
+ )
+ if temp_dir_path:
+ await viking_fs.delete_temp(temp_dir_path, ctx=ctx)
+ finally:
+ await resource_lock.close()
+ return result
+
async def reserve_unique_candidate(
self,
*,
diff --git a/tests/storage/test_semantic_processor_target_preexisting.py b/tests/storage/test_semantic_processor_target_preexisting.py
index 2546509a3..109d33a55 100644
--- a/tests/storage/test_semantic_processor_target_preexisting.py
+++ b/tests/storage/test_semantic_processor_target_preexisting.py
@@ -149,3 +149,23 @@ async def test_sync_diff_reports_target_uris_and_preserves_sidecars(monkeypatch)
)
assert fake_fs.contents["viking://resources/root/.abstract.md"] == "old abstract"
assert fake_fs.deleted_temp == ["viking://temp/import"]
+
+
+async def test_sync_missing_source_never_touches_target(monkeypatch):
+ fake_fs = AsyncMock()
+ fake_fs.exists.return_value = False
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.get_viking_fs",
+ lambda: fake_fs,
+ )
+
+ with pytest.raises(FileNotFoundError, match="refusing to sync"):
+ await SemanticProcessor()._sync_topdown_recursive(
+ "viking://temp/missing",
+ "viking://resources/root",
+ lock=NO_LOCK,
+ )
+
+ fake_fs.ls.assert_not_awaited()
+ fake_fs.rm.assert_not_awaited()