Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(task): 修复 add-resource 任务重启后无法恢复 (#3334)
Qin Haojie1d4bf132ed11
diff --git a/openviking/service/core.py b/openviking/service/core.pyindex d78ced348..0351a1039 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -28,9 +28,9 @@ from openviking.session import create_session_compressorfrom openviking.storage import VikingDBManagerfrom openviking.storage.collection_schemas import init_context_collectionfrom openviking.storage.index_consistency import check_index_consistency+from openviking.storage.queuefs.add_resource_processor import AddResourceProcessorfrom openviking.storage.queuefs.queue_manager import QueueManager, init_queue_managerfrom openviking.storage.queuefs.session_commit_processor import SessionCommitProcessor-from openviking.storage.queuefs.understanding_parse_processor import UnderstandingParseProcessorfrom openviking.storage.transaction import LockManager, init_lock_managerfrom openviking.storage.viking_fs import VikingFS, init_viking_fsfrom openviking.utils.agfs_utils import (@@ -416,12 +416,12 @@ class OpenVikingService:)if self._queue_manager:- external_parse_processor = UnderstandingParseProcessor(- self._resource_processor,- resource_memory_link_service=self._resource_memory_link_service,+ external_parse_processor = AddResourceProcessor(+ self._resource_service,+ asyncio.get_running_loop(),)self._queue_manager.get_queue(- self._queue_manager.EXTERNAL_PARSE,+ self._queue_manager.ADD_RESOURCE,dequeue_handler=external_parse_processor,allow_create=True,)diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.pyindex dce540802..44ec34c62 100644--- a/openviking/service/resource_service.py+++ b/openviking/service/resource_service.py@@ -15,6 +15,7 @@ from collections.abc import Callablefrom dataclasses import dataclassfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Dict, List, Optional+from uuid import uuid4import httpx@@ -46,7 +47,6 @@ from openviking.telemetry.resource_summary import (build_queue_status_payload,record_resource_wait_metrics,register_wait_telemetry,- summarize_queue_errors,unregister_wait_telemetry,)from openviking.utils import is_git_repo_url, parse_code_hosting_url@@ -100,6 +100,7 @@ _ADD_RESOURCE_ARGS_RESERVED_FIELDS = frozenset("create_parent","telemetry","request_validator",+ "defer_post_processing",})@@ -325,7 +326,7 @@ class ResourceService:raise NotInitializedError("VikingFS")async def close_background_tasks(self) -> None:- """Cancel in-flight background resource ingestion tasks during service shutdown."""+ """Cancel in-flight connector monitoring tasks during service shutdown."""if not self._background_tasks:returntasks = list(self._background_tasks)@@ -334,6 +335,129 @@ class ResourceService:await asyncio.gather(*tasks, return_exceptions=True)self._background_tasks.clear()+ async def _enqueue_add_resource_job(+ self,+ msg: Any,+ *,+ resource_lock: LockLease = NO_LOCK,+ ) -> Any:+ """Persist a job before its TaskRecord so a crash cannot orphan the task."""+ from openviking.service.task_tracker import get_task_tracker+ from openviking.storage.queuefs import QueueManager, get_queue_manager++ try:+ await get_queue_manager().enqueue(QueueManager.ADD_RESOURCE, msg.to_dict())+ await resource_lock.handoff()+ except BaseException:+ await resource_lock.close()+ raise++ tracker = get_task_tracker()+ task = await tracker.create(+ "add_resource",+ resource_id=msg.root_uri,+ account_id=msg.account_id,+ user_id=msg.user_id,+ task_id=msg.task_id,+ )+ await tracker.update_stage(+ task.task_id,+ "queued",+ account_id=msg.account_id,+ user_id=msg.user_id,+ )+ return task++ async def execute_add_resource_job(+ self,+ msg: Any,+ *,+ ctx: RequestContext,+ resource_lock: Optional[LockLease],+ stage_callback: Callable[[str], Any],+ ) -> Dict[str, Any]:+ """Execute one durable add-resource job inside its QueueFS consumer."""+ resource_lock = resource_lock or NO_LOCK+ if msg.prepared is None:+ return await self.add_resource(+ path=msg.path,+ ctx=ctx,+ to=msg.root_uri,+ reason=msg.reason,+ instruction=msg.instruction,+ wait=True,+ timeout=msg.timeout,+ build_index=msg.build_index,+ summarize=msg.summarize,+ watch_interval=msg.watch_interval,+ skip_watch_management=msg.skip_watch_management,+ allow_local_path_resolution=msg.allow_local_path_resolution,+ enforce_public_remote_targets=msg.enforce_public_remote_targets,+ resource_lock=resource_lock,+ stage_callback=stage_callback,+ strict=msg.strict,+ source_name=msg.source_name,+ ignore_dirs=msg.ignore_dirs,+ include=msg.include,+ exclude=msg.exclude,+ directly_upload_media=msg.directly_upload_media,+ preserve_structure=msg.preserve_structure,+ create_parent=msg.create_parent,+ args=msg.args,+ )++ telemetry_id = get_current_telemetry().telemetry_id+ request_wait_tracker = get_request_wait_tracker()+ request_wait_tracker.register_request(telemetry_id)+ try:+ stage_result = stage_callback("processing_queue")+ if inspect.isawaitable(stage_result):+ await stage_result+ result = await self._resource_processor.finish_prepared_resource(+ msg.prepared,+ ctx=ctx,+ resource_lock=resource_lock,+ summarize=msg.summarize,+ build_index=msg.build_index,+ )+ await request_wait_tracker.wait_for_request(+ telemetry_id,+ timeout=msg.timeout,+ )+ status = request_wait_tracker.build_queue_status(telemetry_id)+ result["queue_status"] = status+ await self._link_resource_reason_memory(+ result=result,+ ctx=ctx,+ reason=msg.reason,+ source_name=msg.source_name,+ timeout=msg.timeout,+ )+ return result+ except TimeoutError as exc:+ raise DeadlineExceededError("queue processing", msg.timeout) from exc+ finally:+ request_wait_tracker.cleanup(telemetry_id)+ unregister_wait_telemetry(telemetry_id)++ async def reacquire_add_resource_job_lock(+ self,+ root_uri: str,+ ctx: RequestContext,+ ) -> LockLease:+ """Acquire a fresh lock when a recovered job's old handoff was released."""+ if not self._resource_processor or not self._viking_fs:+ raise NotInitializedError("ResourceProcessor")+ from openviking.storage.transaction import get_lock_manager++ dst_path = self._viking_fs._uri_to_path(root_uri, ctx=ctx)+ return await self._resource_processor.acquire_resource_lock(+ get_lock_manager(),+ dst_path,+ uri=root_uri,+ timeout=0.0,+ )+async def enqueue_git_add_resource(self,path: str,@@ -365,7 +489,7 @@ class ResourceService:create_parent=bool(kwargs.get("create_parent", False)),)- from openviking.service.task_tracker import get_task_tracker+ from openviking.storage.queuefs.add_resource_msg import AddResourceMsgresource_lock: LockLease = NO_LOCKtry:@@ -385,26 +509,23 @@ class ResourceService:source_info=source_info,)- task_tracker = get_task_tracker()- task = await task_tracker.create(- "add_resource",- resource_id=root_uri,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- await task_tracker.update_stage(- task.task_id,- "queued",+ task_id = str(uuid4())+ lock_handoff = resource_lock.to_handoff()+ processor_args = {+ key: value+ for key, value in kwargs.items()+ if key not in _ADD_RESOURCE_ARGS_RESERVED_FIELDS+ }+ msg = AddResourceMsg(+ task_id=task_id,+ path=path,+ root_uri=root_uri,+ telemetry_id=get_current_telemetry().telemetry_id or None,account_id=ctx.account_id,user_id=ctx.user.user_id,- )-- add_kwargs = dict(- kwargs,- path=path,- ctx=ctx,- to=root_uri,- parent=None,+ role=str(ctx.role),+ actor_peer_id=ctx.actor_peer_id,+ lock_handoff=lock_handoff.to_dict() if lock_handoff else None,reason=reason,instruction=instruction,timeout=timeout,@@ -414,18 +535,18 @@ class ResourceService:skip_watch_management=skip_watch_management,allow_local_path_resolution=allow_local_path_resolution,enforce_public_remote_targets=enforce_public_remote_targets,+ strict=bool(kwargs.get("strict", False)),+ ignore_dirs=kwargs.get("ignore_dirs"),+ include=kwargs.get("include"),+ exclude=kwargs.get("exclude"),+ directly_upload_media=bool(kwargs.get("directly_upload_media", True)),+ preserve_structure=kwargs.get("preserve_structure"),+ create_parent=bool(kwargs.get("create_parent", False)),+ source_name=source_name,+ args=self._sanitize_watch_processor_kwargs(processor_args),)- background = asyncio.create_task(- self._run_add_resource_task(- task.task_id,- ctx=ctx,- add_kwargs=add_kwargs,- resource_lock=resource_lock,- )- )+ task = await self._enqueue_add_resource_job(msg, resource_lock=resource_lock)resource_lock = NO_LOCK- self._background_tasks.add(background)- background.add_done_callback(self._background_tasks.discard)return {"status": "success","root_uri": root_uri,@@ -435,81 +556,6 @@ class ResourceService:await resource_lock.close()raise- async def _run_add_resource_task(- self,- task_id: str,- *,- ctx: RequestContext,- add_kwargs: Dict[str, Any],- resource_lock: LockLease,- ) -> None:- from openviking.service.task_tracker import get_task_tracker-- task_tracker = get_task_tracker()-- async def _set_stage(stage: str) -> None:- await task_tracker.update_stage(- task_id,- stage,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )-- try:- await task_tracker.start(- task_id,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- stage="queued",- )- result = await self.add_resource(- wait=True,- resource_lock=resource_lock,- stage_callback=_set_stage,- **add_kwargs,- )- if result.get("status") == "error":- errors = result.get("errors") or ["resource processing failed"]- await task_tracker.fail(- task_id,- "; ".join(str(error) for error in errors),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- return- queue_errors = summarize_queue_errors(result.get("queue_status"))- if queue_errors:- await task_tracker.fail(- task_id,- "queue processing failed: " + "; ".join(queue_errors),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- return- await task_tracker.complete(- task_id,- result,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- except asyncio.CancelledError:- await task_tracker.fail(- task_id,- "background resource ingestion cancelled",- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- raise- except Exception as exc:- await task_tracker.fail(- task_id,- str(exc),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- finally:- await resource_lock.close()-async def _plan_resource_target(self,*,@@ -731,7 +777,8 @@ class ResourceService:telemetry = get_current_telemetry()telemetry_id = register_wait_telemetry(wait)request_wait_tracker = get_request_wait_tracker()- monitor_started = False+ job_enqueued = False+ deferred_lock: LockLease = NO_LOCKif telemetry_id:request_wait_tracker.register_request(telemetry_id)watch_manager = self._get_watch_manager()@@ -763,11 +810,7 @@ class ResourceService:and not allow_local_path_resolutionand self._resource_processor is not None):- from openviking.service.task_tracker import get_task_tracker- from openviking.storage.queuefs import QueueManager, get_queue_manager- from openviking.storage.queuefs.understanding_parse_msg import (- UnderstandingParseMsg,- )+ from openviking.storage.queuefs.add_resource_msg import AddResourceMsgsource_name = kwargs.get("source_name")source_info = _ResourceSourceInfo(@@ -820,67 +863,40 @@ class ResourceService:else:lock_lease = await _reserve_tree(root_uri)- task_tracker = get_task_tracker()- task = None- try:- task = await task_tracker.create(- "add_resource",- resource_id=root_uri,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- await task_tracker.update_stage(- task.task_id,- "queued_external_parse",- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )-- lock_handoff = lock_lease.to_handoff()- msg = UnderstandingParseMsg(- task_id=task.task_id,- telemetry_id=telemetry_id or None,- path=path,- root_uri=root_uri,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- role=str(ctx.role),- actor_peer_id=ctx.actor_peer_id,- reason=reason,- instruction=instruction,- build_index=build_index,- summarize=summarize,- strict=bool(kwargs.get("strict", False)),- ignore_dirs=kwargs.get("ignore_dirs"),- include=kwargs.get("include"),- exclude=kwargs.get("exclude"),- directly_upload_media=bool(kwargs.get("directly_upload_media", True)),- allow_local_path_resolution=allow_local_path_resolution,- enforce_public_remote_targets=enforce_public_remote_targets,- args=normalized_args.processor_kwargs,- source_name=source_name,- lock_handoff=lock_handoff.to_dict() if lock_handoff else None,- )- qm = get_queue_manager()- await qm.enqueue(QueueManager.EXTERNAL_PARSE, msg.to_dict())- await lock_lease.handoff()- monitor_started = True- except Exception as exc:- with contextlib.suppress(Exception):- await lock_lease.close()-- if task is not None:- with contextlib.suppress(Exception):- await task_tracker.fail(- task.task_id,- str(exc),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )-- raise+ lock_handoff = lock_lease.to_handoff()+ msg = AddResourceMsg(+ task_id=str(uuid4()),+ telemetry_id=telemetry_id or None,+ path=path,+ root_uri=root_uri,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ role=str(ctx.role),+ actor_peer_id=ctx.actor_peer_id,+ reason=reason,+ instruction=instruction,+ timeout=timeout,+ build_index=build_index,+ summarize=summarize,+ strict=bool(kwargs.get("strict", False)),+ ignore_dirs=kwargs.get("ignore_dirs"),+ include=kwargs.get("include"),+ exclude=kwargs.get("exclude"),+ directly_upload_media=bool(kwargs.get("directly_upload_media", True)),+ preserve_structure=kwargs.get("preserve_structure"),+ create_parent=bool(kwargs.get("create_parent", False)),+ allow_local_path_resolution=allow_local_path_resolution,+ enforce_public_remote_targets=enforce_public_remote_targets,+ args=self._sanitize_watch_processor_kwargs(normalized_args.processor_kwargs),+ source_name=source_name,+ lock_handoff=lock_handoff.to_dict() if lock_handoff else None,+ skip_watch_management=True,+ )+ task = await self._enqueue_add_resource_job(msg, resource_lock=lock_lease)+ lock_lease = NO_LOCK+ job_enqueued = Truelogger.info(- "[ResourceService] Enqueued UnderstandingParseMsg task_id=%s root_uri=%s",+ "[ResourceService] Enqueued AddResourceMsg task_id=%s root_uri=%s",task.task_id,root_uri,)@@ -917,11 +933,14 @@ class ResourceService:summarize=summarize,stage_callback=stage_callback,allow_local_path_resolution=allow_local_path_resolution,+ defer_post_processing=not wait,**kwargs,)if result.get("status") == "error":return result+ prepared = result.pop("_post_process", None)+ deferred_lock = result.pop("_resource_lock", NO_LOCK)if wait:if stage_callback is not None:stage_result = stage_callback("processing_queue")@@ -969,6 +988,47 @@ class ResourceService:root_uri=result.get("root_uri"),)telemetry.set("queue.wait.duration_ms", queue_wait_duration_ms)+ if not wait:+ from openviking.storage.queuefs.add_resource_msg import AddResourceMsg++ root_uri = result.get("root_uri", "")+ if not isinstance(prepared, dict):+ raise InternalError("Deferred resource processing payload is missing")+ lock_handoff = deferred_lock.to_handoff()+ msg = AddResourceMsg(+ task_id=str(uuid4()),+ root_uri=root_uri,+ prepared=prepared,+ telemetry_id=telemetry_id or None,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ role=str(ctx.role),+ actor_peer_id=ctx.actor_peer_id,+ lock_handoff=lock_handoff.to_dict() if lock_handoff else None,+ reason=reason,+ instruction=instruction,+ timeout=timeout,+ build_index=build_index,+ summarize=summarize,+ strict=bool(kwargs.get("strict", False)),+ ignore_dirs=kwargs.get("ignore_dirs"),+ include=kwargs.get("include"),+ exclude=kwargs.get("exclude"),+ directly_upload_media=bool(kwargs.get("directly_upload_media", True)),+ preserve_structure=kwargs.get("preserve_structure"),+ create_parent=bool(kwargs.get("create_parent", False)),+ allow_local_path_resolution=allow_local_path_resolution,+ enforce_public_remote_targets=enforce_public_remote_targets,+ source_name=kwargs.get("source_name"),+ skip_watch_management=True,+ )+ task = await self._enqueue_add_resource_job(+ msg,+ resource_lock=deferred_lock,+ )+ deferred_lock = NO_LOCK+ result["task_id"] = task.task_id+ job_enqueued = Trueawait self._manage_watch_if_needed(watch_manager=watch_manager,skip_watch_management=skip_watch_management,@@ -992,48 +1052,6 @@ class ResourceService:source_name=kwargs.get("source_name"),timeout=timeout,)- if not wait:- from openviking.service.task_tracker import get_task_tracker-- task_tracker = get_task_tracker()- root_uri = result.get("root_uri", "")- task = await task_tracker.create(- "add_resource",- resource_id=root_uri,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- result["task_id"] = task.task_id- if telemetry_id:- monitor_started = True- background = asyncio.create_task(- self._monitor_resource_queue_then_link_memory(- task.task_id,- telemetry_id,- ctx,- root_uri=root_uri,- reason=reason,- source_name=kwargs.get("source_name"),- timeout=timeout,- )- )- self._background_tasks.add(background)- background.add_done_callback(self._background_tasks.discard)- else:- monitor_started = True- background = asyncio.create_task(- self._monitor_resource_queue_then_link_memory(- task.task_id,- None,- ctx,- root_uri=root_uri,- reason=reason,- source_name=kwargs.get("source_name"),- timeout=timeout,- )- )- self._background_tasks.add(background)- background.add_done_callback(self._background_tasks.discard)return resultexcept Exception as exc:telemetry.set_error(@@ -1047,9 +1065,11 @@ class ResourceService:"resource.request.duration_ms",round((time.perf_counter() - request_start) * 1000, 3),)- if wait or not telemetry_id or not monitor_started:+ if wait or not telemetry_id or not job_enqueued:get_request_wait_tracker().cleanup(telemetry_id)unregister_wait_telemetry(telemetry_id)+ if deferred_lock.active:+ await deferred_lock.close()async def _link_resource_reason_memory(self,@@ -1080,66 +1100,6 @@ class ResourceService:logger.warning("[ResourceService] Failed to link resource reason memory: %s", exc)result.setdefault("warnings", []).append(f"Memory linking failed: {exc}")- async def _monitor_resource_queue_then_link_memory(- self,- task_id: str,- telemetry_id: Optional[str],- ctx: RequestContext,- *,- root_uri: str,- reason: str,- source_name: Optional[str],- timeout: Optional[float],- ) -> None:- from openviking.service.task_tracker import get_task_tracker-- task_tracker = get_task_tracker()- request_wait_tracker = get_request_wait_tracker()- await task_tracker.start(task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)- try:- if telemetry_id:- await request_wait_tracker.wait_for_request(telemetry_id)- status = request_wait_tracker.build_queue_status(telemetry_id)- else:- status = build_queue_status_payload(- await get_queue_manager().wait_complete(timeout=timeout)- )- errors = sum(int(group.get("error_count", 0) or 0) for group in status.values())- if errors:- await task_tracker.fail(- task_id,- f"queue processing failed: {status}",- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- return-- result: Dict[str, Any] = {"root_uri": root_uri, "queue_status": status}- await self._link_resource_reason_memory(- result=result,- ctx=ctx,- reason=reason,- source_name=source_name,- timeout=timeout,- )- await task_tracker.complete(- task_id,- result,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- except Exception as exc:- await task_tracker.fail(- task_id,- str(exc),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- finally:- if telemetry_id:- request_wait_tracker.cleanup(telemetry_id)- unregister_wait_telemetry(telemetry_id)-async def _monitor_queue_processing(self,task_id: str,@@ -1297,7 +1257,9 @@ class ResourceService:"wait=true (Connector imports run asynchronously; poll the returned task_id)")if reason:- unsupported.append("reason (Connector imports cannot preserve resource-reason semantics)")+ unsupported.append(+ "reason (Connector imports cannot preserve resource-reason semantics)"+ )if instruction:unsupported.append("instruction")if not build_index:diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.pyindex c020497b4..40e5b0a9f 100644--- a/openviking/service/task_tracker.py+++ b/openviking/service/task_tracker.py@@ -39,6 +39,9 @@ class TaskStatus(str, Enum):FAILED = "failed"+_TERMINAL_STATUSES = (TaskStatus.COMPLETED, TaskStatus.FAILED)++@dataclassclass TaskRecord:"""Immutable snapshot of an async task."""@@ -333,7 +336,7 @@ class TaskTracker:"""Transition task to RUNNING."""async with self._async_lock:task = await self._load_for_update(task_id, account_id, user_id)- if task:+ if task and task.status not in _TERMINAL_STATUSES:task.status = TaskStatus.RUNNINGif stage is not None:task.stage = stage@@ -352,7 +355,7 @@ class TaskTracker:"""Update task stage without changing its lifecycle status."""async with self._async_lock:task = await self._load_for_update(task_id, account_id, user_id)- if task:+ if task and task.status not in _TERMINAL_STATUSES:task.stage = stagetask.updated_at = time.time()await self._store.update(task)@@ -367,9 +370,10 @@ class TaskTracker:user_id: Optional[str] = None,) -> None:"""Transition task to COMPLETED with optional result."""+ transitioned = Falseasync with self._async_lock:task = await self._load_for_update(task_id, account_id, user_id)- if task:+ if task and task.status not in _TERMINAL_STATUSES:task.status = TaskStatus.COMPLETEDtask.stage = "completed"task.result = result@@ -377,7 +381,9 @@ class TaskTracker:await self._store.update(task)with self._lock:self._tasks[task.task_id] = task- logger.info("[TaskTracker] Task %s completed", task_id)+ transitioned = True+ if transitioned:+ logger.info("[TaskTracker] Task %s completed", task_id)async def fail(self,@@ -387,9 +393,10 @@ class TaskTracker:user_id: Optional[str] = None,) -> None:"""Transition task to FAILED with sanitized error."""+ transitioned = Falseasync with self._async_lock:task = await self._load_for_update(task_id, account_id, user_id)- if task:+ if task and task.status not in _TERMINAL_STATUSES:task.status = TaskStatus.FAILEDtask.stage = "failed"task.error = _sanitize_error(error)@@ -397,7 +404,9 @@ class TaskTracker:await self._store.update(task)with self._lock:self._tasks[task.task_id] = task- logger.warning("[TaskTracker] Task %s failed: %s", task_id, _sanitize_error(error))+ transitioned = True+ if transitioned:+ logger.warning("[TaskTracker] Task %s failed: %s", task_id, _sanitize_error(error))async def get(self,diff --git a/openviking/storage/queuefs/add_resource_msg.py b/openviking/storage/queuefs/add_resource_msg.pynew file mode 100644index 000000000..16dd5c624--- /dev/null+++ b/openviking/storage/queuefs/add_resource_msg.py@@ -0,0 +1,111 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Persistent add-resource queue message (legacy ExternalParse queue payload)."""++from dataclasses import asdict, dataclass, field+from typing import Any, Dict, Optional+++@dataclass(kw_only=True)+class AddResourceMsg:+ task_id: str+ root_uri: str+ account_id: str+ user_id: str+ role: str+ path: str = ""+ telemetry_id: Optional[str] = None+ prepared: Optional[Dict[str, Any]] = None+ lock_handoff: Optional[Dict[str, Any]] = None+ actor_peer_id: Optional[str] = None+ reason: str = ""+ instruction: str = ""+ timeout: Optional[float] = None+ build_index: bool = True+ summarize: bool = False+ strict: bool = False+ ignore_dirs: Optional[str] = None+ include: Optional[str] = None+ exclude: Optional[str] = None+ directly_upload_media: bool = True+ preserve_structure: Optional[bool] = None+ create_parent: bool = False+ allow_local_path_resolution: bool = True+ enforce_public_remote_targets: bool = False+ args: Dict[str, Any] = field(default_factory=dict)+ lock_handoff_retry: int = 0+ source_name: Optional[str] = None+ watch_interval: float = 0+ skip_watch_management: bool = True++ def to_dict(self) -> Dict[str, Any]:+ data = asdict(self)+ if self.prepared is not None:+ data["args"] = {}+ return data++ @classmethod+ def from_dict(cls, data: Dict[str, Any]) -> "AddResourceMsg":+ if not isinstance(data, dict) or not data:+ raise ValueError("Data dictionary is empty")+ task_id = data.get("task_id")+ path = data.get("path")+ root_uri = data.get("root_uri")+ prepared = data.get("prepared") if isinstance(data.get("prepared"), dict) else None+ args = dict(data.get("args", {})) if isinstance(data.get("args"), dict) else {}+ legacy_retry = args.pop("_lock_handoff_retry", 0)+ try:+ lock_handoff_retry = max(0, int(data.get("lock_handoff_retry", legacy_retry) or 0))+ except (TypeError, ValueError):+ lock_handoff_retry = 0+ if prepared is not None:+ args.clear()+ if not task_id or (not path and not prepared) or not root_uri:+ missing = []+ if not task_id:+ missing.append("task_id")+ if not path and not prepared:+ missing.append("path or prepared")+ if not root_uri:+ missing.append("root_uri")+ raise ValueError(f"Missing required fields: {missing}")++ return cls(+ task_id=str(task_id),+ path=str(path or ""),+ root_uri=str(root_uri),+ account_id=str(data.get("account_id", "default")),+ user_id=str(data.get("user_id", "default")),+ role=str(data.get("role", "root")),+ actor_peer_id=data.get("actor_peer_id"),+ telemetry_id=str(data.get("telemetry_id"))+ if isinstance(data.get("telemetry_id"), str)+ else None,+ lock_handoff=data.get("lock_handoff")+ if isinstance(data.get("lock_handoff"), dict)+ else None,+ reason=str(data.get("reason", "")),+ instruction=str(data.get("instruction", "")),+ timeout=float(data["timeout"]) if data.get("timeout") is not None else None,+ build_index=bool(data.get("build_index", True)),+ summarize=bool(data.get("summarize", False)),+ strict=bool(data.get("strict", False)),+ ignore_dirs=data.get("ignore_dirs"),+ include=data.get("include"),+ exclude=data.get("exclude"),+ directly_upload_media=bool(data.get("directly_upload_media", True)),+ preserve_structure=(+ bool(data["preserve_structure"])+ if data.get("preserve_structure") is not None+ else None+ ),+ create_parent=bool(data.get("create_parent", False)),+ allow_local_path_resolution=bool(data.get("allow_local_path_resolution", True)),+ enforce_public_remote_targets=bool(data.get("enforce_public_remote_targets", False)),+ args=args,+ lock_handoff_retry=lock_handoff_retry,+ source_name=data.get("source_name"),+ prepared=prepared,+ watch_interval=float(data.get("watch_interval", 0) or 0),+ skip_watch_management=bool(data.get("skip_watch_management", True)),+ )diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.pynew file mode 100644index 000000000..d5e3ec6e5--- /dev/null+++ b/openviking/storage/queuefs/add_resource_processor.py@@ -0,0 +1,204 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Durable add-resource consumer on the legacy ExternalParse queue."""++import asyncio+import concurrent.futures+import json+from contextlib import suppress+from typing import Any, Dict, Optional++from openviking.observability.context import bind_execution_context+from openviking.server.identity import RequestContext, Role+from openviking.service.task_tracker import TaskStatus, get_task_tracker+from openviking.storage.queuefs.add_resource_msg import AddResourceMsg+from openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.telemetry import bind_telemetry, resolve_telemetry+from openviking.telemetry.resource_summary import summarize_queue_errors+from openviking_cli.session.user_id import UserIdentifier+from openviking_cli.utils.logger import get_logger++logger = get_logger(__name__)+++class AddResourceProcessor(DequeueHandlerBase):+ """Own an add-resource task until it reaches a terminal state and can be ACKed."""++ def __init__(+ self,+ resource_service: Any,+ service_loop: asyncio.AbstractEventLoop,+ ):+ self._resource_service = resource_service+ self._service_loop = service_loop++ async def _load_lock(self, msg: AddResourceMsg, ctx: RequestContext) -> Any:+ if msg.lock_handoff is None:+ return None+ from openviking.storage.transaction.lock_lease import LockHandoffRef, OwnedLockLease++ ref = LockHandoffRef.from_value(msg.lock_handoff)+ if ref is None:+ raise ValueError("Invalid lock_handoff")+ try:+ return await OwnedLockLease.from_handoff(ref)+ except Exception as handoff_error:+ try:+ return await self._resource_service.reacquire_add_resource_job_lock(+ msg.root_uri,+ ctx,+ )+ except Exception:+ raise handoff_error++ async def _requeue_lock_handoff(self, msg: AddResourceMsg, exc: Exception) -> bool:+ if msg.lock_handoff_retry >= 2:+ return False++ from openviking.storage.queuefs import QueueManager, get_queue_manager++ payload = msg.to_dict()+ payload["lock_handoff_retry"] = msg.lock_handoff_retry + 1+ await get_queue_manager().enqueue(QueueManager.ADD_RESOURCE, payload)+ logger.warning(+ "[AddResource] Requeued task %s after lock handoff failure: %s",+ msg.task_id,+ exc,+ )+ self.report_requeue()+ self.report_success()+ return True++ async def _process(self, msg: AddResourceMsg, data: Dict[str, Any]) -> None:+ ctx = RequestContext(+ user=UserIdentifier(msg.account_id, msg.user_id),+ role=Role(msg.role),+ actor_peer_id=msg.actor_peer_id,+ )+ tracker = get_task_tracker()+ task = await tracker.create(+ "add_resource",+ resource_id=msg.root_uri,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ task_id=msg.task_id,+ )+ if task.status in (TaskStatus.COMPLETED, TaskStatus.FAILED):+ self.report_success()+ return None++ resource_lock = None+ try:+ resource_lock = await self._load_lock(msg, ctx)+ except Exception as exc:+ if await self._requeue_lock_handoff(msg, exc):+ return None+ await tracker.fail(+ msg.task_id,+ f"Invalid lock_handoff: {exc}",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_error(f"Invalid lock_handoff: {exc}", data)+ return None++ telemetry_id = msg.telemetry_id or ""+ telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None+ if telemetry is None:+ from openviking.telemetry.operation import OperationTelemetry++ telemetry = OperationTelemetry(operation="add_resource_job", enabled=False)+ if telemetry_id:+ telemetry.telemetry_id = telemetry_id++ async def _set_stage(stage: str) -> None:+ await tracker.update_stage(+ msg.task_id,+ stage,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )++ with bind_execution_context(), bind_telemetry(telemetry):+ try:+ await tracker.start(+ msg.task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ stage="queued",+ )+ result = await self._resource_service.execute_add_resource_job(+ msg,+ ctx=ctx,+ resource_lock=resource_lock,+ stage_callback=_set_stage,+ )+ if result.get("status") == "error":+ errors = result.get("errors") or ["resource processing failed"]+ await tracker.fail(+ msg.task_id,+ "; ".join(str(error) for error in errors),+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_error("resource processing failed", data)+ return None+ queue_errors = summarize_queue_errors(result.get("queue_status"))+ if queue_errors:+ await tracker.fail(+ msg.task_id,+ "queue processing failed: " + "; ".join(queue_errors),+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_error("queue processing failed", data)+ return None+ await tracker.complete(+ msg.task_id,+ result,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_success()+ return None+ except asyncio.CancelledError:+ # Leave both task and QueueFS message active; RecoverStale owns restart recovery.+ raise+ except Exception as exc:+ await tracker.fail(+ msg.task_id,+ str(exc),+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_error(str(exc), data)+ return None+ finally:+ with suppress(Exception):+ if resource_lock is not None:+ await resource_lock.close()++ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ if not data:+ return None+ try:+ if not isinstance(data, dict):+ raise ValueError("Queue message must be an object")+ payload = data.get("data", data)+ if isinstance(payload, str):+ payload = json.loads(payload)+ msg = AddResourceMsg.from_dict(payload)+ except (json.JSONDecodeError, TypeError, ValueError) as exc:+ self.report_error(str(exc), data)+ return None++ future: concurrent.futures.Future[None] = asyncio.run_coroutine_threadsafe(+ self._process(msg, data),+ self._service_loop,+ )+ try:+ await asyncio.wrap_future(future)+ except asyncio.CancelledError:+ future.cancel()+ raise+ return Nonediff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 1cfd6d4c9..3de4efc62 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -69,7 +69,8 @@ class QueueManager:# Standard queue namesEMBEDDING = "Embedding"SEMANTIC = "Semantic"- EXTERNAL_PARSE = "ExternalParse"+ # Keep the on-disk queue name stable so pre-upgrade jobs remain recoverable.+ ADD_RESOURCE = "ExternalParse"SESSION_COMMIT = "SessionCommit"def __init__(@@ -157,7 +158,7 @@ class QueueManager:if queue.name == self.EMBEDDING:max_concurrent = self._max_concurrent_embedding- elif queue.name in {self.EXTERNAL_PARSE, self.SESSION_COMMIT}:+ elif queue.name in {self.ADD_RESOURCE, self.SESSION_COMMIT}:max_concurrent = self._max_concurrent_external_parseelse:max_concurrent = self._max_concurrent_semanticdiff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex e26c51d91..016a03e15 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -748,19 +748,19 @@ class SemanticProcessor(DequeueHandlerBase):lock: LockLease = NO_LOCK,) -> DiffResult:viking_fs = get_viking_fs()+ if not await viking_fs.exists(root_uri, ctx=ctx):+ raise FileNotFoundError(+ f"Semantic source no longer exists; refusing to sync into {target_uri}: {root_uri}"+ )diff = DiffResult()lock_handle = lock.handleasync def list_children(dir_uri: str) -> Tuple[Dict[str, str], Dict[str, str]]:files: Dict[str, str] = {}dirs: Dict[str, str] = {}- try:- entries = await viking_fs.ls(- dir_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx- )- except Exception as e:- logger.error(f"[SyncDiff] Failed to list {dir_uri}: {e}")- return files, dirs+ entries = await viking_fs.ls(+ dir_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx+ )for entry in entries:name = entry.get("name", "")diff --git a/openviking/storage/queuefs/understanding_parse_msg.py b/openviking/storage/queuefs/understanding_parse_msg.pydeleted file mode 100644index 12ea5c731..000000000--- a/openviking/storage/queuefs/understanding_parse_msg.py+++ /dev/null@@ -1,155 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-"""UnderstandingParseMsg: External parse queue message for UnderstandingAPI."""--import json-from dataclasses import asdict, dataclass, field-from datetime import datetime-from typing import Any, Dict, Optional-from uuid import uuid4---@dataclass-class UnderstandingParseMsg:- id: str- task_id: str- telemetry_id: Optional[str]- path: str- root_uri: str- lock_handoff: Optional[Dict[str, Any]] = None- status: str = "pending"- timestamp: int = int(datetime.now().timestamp())- account_id: str = "default"- user_id: str = "default"- role: str = "root"- actor_peer_id: Optional[str] = None- reason: str = ""- instruction: str = ""- build_index: bool = True- summarize: bool = False- strict: bool = False- ignore_dirs: Optional[str] = None- include: Optional[str] = None- exclude: Optional[str] = None- directly_upload_media: bool = True- allow_local_path_resolution: bool = True- enforce_public_remote_targets: bool = False- args: Dict[str, Any] = field(default_factory=dict)- source_name: Optional[str] = None-- def __init__(- self,- *,- task_id: str,- path: str,- root_uri: str,- account_id: str,- user_id: str,- role: str,- actor_peer_id: Optional[str] = None,- telemetry_id: Optional[str] = None,- lock_handoff: Optional[Dict[str, Any]] = None,- reason: str = "",- instruction: str = "",- build_index: bool = True,- summarize: bool = False,- strict: bool = False,- ignore_dirs: Optional[str] = None,- include: Optional[str] = None,- exclude: Optional[str] = None,- directly_upload_media: bool = True,- allow_local_path_resolution: bool = True,- enforce_public_remote_targets: bool = False,- args: Optional[Dict[str, Any]] = None,- source_name: Optional[str] = None,- ):- self.id = str(uuid4())- self.task_id = task_id- self.telemetry_id = telemetry_id- self.path = path- self.root_uri = root_uri- self.account_id = account_id- self.user_id = user_id- self.role = role- self.actor_peer_id = actor_peer_id- self.lock_handoff = lock_handoff- self.reason = reason- self.instruction = instruction- self.build_index = build_index- self.summarize = summarize- self.strict = strict- self.ignore_dirs = ignore_dirs- self.include = include- self.exclude = exclude- self.directly_upload_media = directly_upload_media- self.allow_local_path_resolution = allow_local_path_resolution- self.enforce_public_remote_targets = enforce_public_remote_targets- self.args = args or {}- self.source_name = source_name-- def to_dict(self) -> Dict[str, Any]:- return asdict(self)-- def to_json(self) -> str:- return json.dumps(self.to_dict())-- @classmethod- def from_dict(cls, data: Dict[str, Any]) -> "UnderstandingParseMsg":- if not data:- raise ValueError("Data dictionary is empty")- task_id = data.get("task_id")- path = data.get("path")- root_uri = data.get("root_uri")- if not task_id or not path or not root_uri:- missing = []- if not task_id:- missing.append("task_id")- if not path:- missing.append("path")- if not root_uri:- missing.append("root_uri")- raise ValueError(f"Missing required fields: {missing}")-- obj = cls(- task_id=str(task_id),- path=str(path),- root_uri=str(root_uri),- account_id=str(data.get("account_id", "default")),- user_id=str(data.get("user_id", "default")),- role=str(data.get("role", "root")),- actor_peer_id=data.get("actor_peer_id"),- telemetry_id=str(data.get("telemetry_id"))- if isinstance(data.get("telemetry_id"), str)- else None,- lock_handoff=data.get("lock_handoff")- if isinstance(data.get("lock_handoff"), dict)- else None,- reason=str(data.get("reason", "")),- instruction=str(data.get("instruction", "")),- build_index=bool(data.get("build_index", True)),- summarize=bool(data.get("summarize", False)),- strict=bool(data.get("strict", False)),- ignore_dirs=data.get("ignore_dirs"),- include=data.get("include"),- exclude=data.get("exclude"),- directly_upload_media=bool(data.get("directly_upload_media", True)),- allow_local_path_resolution=bool(data.get("allow_local_path_resolution", True)),- enforce_public_remote_targets=bool(data.get("enforce_public_remote_targets", False)),- args=data.get("args") if isinstance(data.get("args"), dict) else {},- source_name=data.get("source_name"),- )- if data.get("id"):- obj.id = str(data["id"])- if "status" in data:- obj.status = str(data["status"])- if "timestamp" in data:- obj.timestamp = int(data["timestamp"])- return obj-- @classmethod- def from_json(cls, json_str: str) -> "UnderstandingParseMsg":- try:- data = json.loads(json_str)- except json.JSONDecodeError as e:- raise ValueError(f"Invalid JSON string: {e}")- return cls.from_dict(data)diff --git a/openviking/storage/queuefs/understanding_parse_processor.py b/openviking/storage/queuefs/understanding_parse_processor.pydeleted file mode 100644index 38f6e770b..000000000--- a/openviking/storage/queuefs/understanding_parse_processor.py+++ /dev/null@@ -1,344 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-"""UnderstandingParseProcessor: Processes UnderstandingParseMsg messages."""--import asyncio-import json--from contextlib import suppress-from typing import Any, Dict, Optional--from openviking.observability.context import (- bind_execution_context,-)-from openviking.server.identity import RequestContext, Role-from openviking.service.task_tracker import get_task_tracker-from openviking.storage.queuefs.named_queue import DequeueHandlerBase-from openviking.storage.queuefs.understanding_parse_msg import UnderstandingParseMsg-from openviking.telemetry import bind_telemetry, bind_telemetry_stage, resolve_telemetry-from openviking_cli.session.user_id import UserIdentifier-from openviking_cli.utils.logger import get_logger--logger = get_logger(__name__)---class UnderstandingParseProcessor(DequeueHandlerBase):- def __init__(self, resource_processor: Any, resource_memory_link_service: Any = None):- self._resource_processor = resource_processor- self._resource_memory_link_service = resource_memory_link_service- self._background_tasks: set[asyncio.Task] = set()-- async def _monitor_queue_then_link_memory(- self,- *,- task_id: str,- telemetry_id: str,- ctx: RequestContext,- result: Dict[str, Any],- reason: str,- source_name: Optional[str],- ) -> None:- from openviking.telemetry.resource_summary import unregister_wait_telemetry- from openviking.telemetry.request_wait_tracker import get_request_wait_tracker-- task_tracker = get_task_tracker()- request_wait_tracker = get_request_wait_tracker()- try:- request_wait_tracker.register_request(telemetry_id)- await request_wait_tracker.wait_for_request(telemetry_id)- status = request_wait_tracker.build_queue_status(telemetry_id)- errors = sum(int(group.get("error_count", 0) or 0) for group in status.values())- if errors:- await task_tracker.fail(- task_id,- f"queue processing failed: {status}",- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- self.report_error("queue processing failed", {"task_id": task_id})- return-- result["queue_status"] = status- if self._resource_memory_link_service and (reason or "").strip():- root_uri = result.get("root_uri")- if root_uri:- try:- link_result = await self._resource_memory_link_service.on_resource_added(- ctx=ctx,- resource_uri=root_uri,- reason=reason,- source_name=source_name,- timeout=None,- )- result["memory_linking"] = link_result- except Exception as exc:- logger.warning(- "[UnderstandingParse] Failed to link resource reason memory: %s",- exc,- )- result.setdefault("warnings", []).append(f"Memory linking failed: {exc}")-- await task_tracker.complete(- task_id,- result,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- self.report_success()- except Exception as exc:- await task_tracker.fail(- task_id,- str(exc),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- self.report_error(str(exc), {"task_id": task_id})- finally:- request_wait_tracker.cleanup(telemetry_id)- unregister_wait_telemetry(telemetry_id)-- async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:- if not data:- return None-- payload: Dict[str, Any] = data- if isinstance(data, dict) and "data" in data:- inner = data.get("data")- if isinstance(inner, str) and inner.strip():- try:- decoded = json.loads(inner)- if isinstance(decoded, dict):- payload = decoded- except Exception:- payload = data- elif isinstance(inner, dict):- payload = inner- if isinstance(payload, dict) and data.get("id") and not payload.get("id"):- payload["id"] = data["id"]-- msg = UnderstandingParseMsg.from_dict(payload)- ctx = RequestContext(- user=UserIdentifier(msg.account_id, msg.user_id),- role=Role(msg.role),- actor_peer_id=msg.actor_peer_id,- )-- task_tracker = get_task_tracker()- lock_lease = None- if msg.lock_handoff is not None:- try:- from openviking.storage.transaction.lock_lease import (- LockHandoffRef,- OwnedLockLease,- )-- ref = LockHandoffRef.from_value(msg.lock_handoff)- if ref is None:- raise ValueError("Invalid lock_handoff")- lock_lease = await OwnedLockLease.from_handoff(ref)- except Exception as exc:- retry_key = "_lock_handoff_retry"- raw_retry = (msg.args or {}).get(retry_key, 0)- try:- retry_count = int(raw_retry or 0)- except (TypeError, ValueError):- retry_count = 0-- max_retries = 2- if retry_count < max_retries:- from openviking.storage.queuefs import QueueManager, get_queue_manager-- qm = get_queue_manager()- if qm is None:- raise- new_args = dict(msg.args or {})- new_args[retry_key] = retry_count + 1- retry_msg = UnderstandingParseMsg(- task_id=msg.task_id,- telemetry_id=msg.telemetry_id,- path=msg.path,- root_uri=msg.root_uri,- account_id=msg.account_id,- user_id=msg.user_id,- role=msg.role,- actor_peer_id=msg.actor_peer_id,- lock_handoff=msg.lock_handoff,- reason=msg.reason,- instruction=msg.instruction,- build_index=msg.build_index,- summarize=msg.summarize,- strict=msg.strict,- ignore_dirs=msg.ignore_dirs,- include=msg.include,- exclude=msg.exclude,- directly_upload_media=msg.directly_upload_media,- allow_local_path_resolution=msg.allow_local_path_resolution,- enforce_public_remote_targets=msg.enforce_public_remote_targets,- args=new_args,- source_name=msg.source_name,- )- await qm.enqueue(QueueManager.EXTERNAL_PARSE, retry_msg.to_dict())- self.report_requeue()- self.report_success()- return None-- await task_tracker.fail(- msg.task_id,- f"Invalid lock_handoff: {exc}",- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- self.report_error(f"Invalid lock_handoff: {exc}", data)- return None-- logger.info(- "[UnderstandingParse] Dequeued task_id=%s root_uri=%s",- msg.task_id,- msg.root_uri,- )- resource_processor = self._resource_processor- if resource_processor is None:- raise RuntimeError("ResourceProcessor is None")-- telemetry_id = msg.telemetry_id or ""- if telemetry_id:- from openviking.telemetry.request_wait_tracker import get_request_wait_tracker-- get_request_wait_tracker().register_request(telemetry_id)-- telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None- if telemetry_id and telemetry is None:- from openviking.telemetry.operation import OperationTelemetry-- telemetry = OperationTelemetry(operation="noop", enabled=False)- telemetry.telemetry_id = telemetry_id-- with bind_execution_context(), (bind_telemetry(telemetry) if telemetry else suppress()):- try:- await task_tracker.start(- msg.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id- )- await task_tracker.update_stage(- msg.task_id,- "parsing",- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )-- bind_telemetry_stage("parse_understanding")- kwargs: Dict[str, Any] = {}- if lock_lease is not None and getattr(lock_lease, "active", False):- kwargs["resource_lock"] = lock_lease- if msg.enforce_public_remote_targets:- from openviking.utils.network_guard import ensure_public_remote_target-- kwargs.setdefault("request_validator", ensure_public_remote_target)- result = await resource_processor.process_resource(- path=msg.path,- ctx=ctx,- reason=msg.reason,- instruction=msg.instruction,- scope="resources",- to=msg.root_uri,- parent=None,- build_index=msg.build_index,- summarize=msg.summarize,- allow_local_path_resolution=msg.allow_local_path_resolution,- enforce_public_remote_targets=msg.enforce_public_remote_targets,- strict=msg.strict,- source_name=msg.source_name,- ignore_dirs=msg.ignore_dirs,- include=msg.include,- exclude=msg.exclude,- directly_upload_media=msg.directly_upload_media,- args=msg.args,- skip_watch_management=True,- watch_interval=0,- **kwargs,- )-- if result.get("status") == "error":- errors = result.get("errors") or ["resource processing failed"]- await task_tracker.fail(- msg.task_id,- "; ".join(str(error) for error in errors),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- self.report_error("resource processing failed", data)- return None-- if telemetry_id:- await task_tracker.update_stage(- msg.task_id,- "processing_queue",- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- monitor = asyncio.create_task(- self._monitor_queue_then_link_memory(- task_id=msg.task_id,- telemetry_id=telemetry_id,- ctx=ctx,- result=result,- reason=msg.reason,- source_name=msg.source_name,- )- )- self._background_tasks.add(monitor)- monitor.add_done_callback(self._background_tasks.discard)- return None-- if self._resource_memory_link_service and (msg.reason or "").strip():- root_uri = result.get("root_uri")- if root_uri:- try:- link_result = await self._resource_memory_link_service.on_resource_added(- ctx=ctx,- resource_uri=root_uri,- reason=msg.reason,- source_name=msg.source_name,- timeout=None,- )- result["memory_linking"] = link_result- except Exception as exc:- logger.warning(- "[UnderstandingParse] Failed to link resource reason memory: %s",- exc,- )- result.setdefault("warnings", []).append(- f"Memory linking failed: {exc}"- )-- await task_tracker.complete(- msg.task_id,- result,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- self.report_success()- return None- except asyncio.CancelledError:- await task_tracker.fail(- msg.task_id,- "external parse cancelled",- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- self.report_error("external parse cancelled", data)- raise- except Exception as exc:- await task_tracker.fail(- msg.task_id,- str(exc),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- self.report_error(str(exc), data)- return None- finally:- try:- if lock_lease is not None:- await lock_lease.close()- except Exception:- passdiff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex 7731a27e7..74c64f04f 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -135,6 +135,7 @@ class ResourceProcessor:"errors": [],"source_path": None,}+ defer_post_processing = bool(kwargs.pop("defer_post_processing", False))preacquired_lock = kwargs.pop("resource_lock", NO_LOCK) or NO_LOCKtelemetry = get_current_telemetry()@@ -360,56 +361,27 @@ class ResourceProcessor:except Exception:pass- # ============ Phase 4: Optional Steps ============- build_index = kwargs.get("build_index", True)- temp_uri_for_summarize = temp_uri or parse_result.temp_dir_path- should_summarize = summarize or build_index- if should_summarize:- skip_vec = not build_index- is_code_repo = parse_result.source_format == "repository"- try:- stage_start = time.perf_counter()- stage_status = "ok"- with telemetry.measure("resource.summarize"):- summary_result = await self._get_summarizer().summarize(- resource_uris=[result["root_uri"]],- ctx=ctx,- skip_vectorization=skip_vec,- lock=resource_lock,- temp_uris=[temp_uri_for_summarize],- is_code_repo=is_code_repo,- target_preexisting=target_preexisting,- **kwargs,- )- if (- resource_lock.active- and summary_result.get("status") == "success"- and summary_result.get("enqueued_count", 0) > 0- ):- await resource_lock.handoff()- resource_lock = NO_LOCK- except Exception as e:- logger.error(f"Summarization failed: {e}")- result["warnings"] = result.get("warnings", []) + [f"Summarization failed: {e}"]- stage_status = "error"- finally:- try:- ResourceIngestionEventDataSource.record_stage(- stage="summarize",- status=str(stage_status),- duration_seconds=float(time.perf_counter() - stage_start),- account_id=getattr(ctx, "account_id", None),- )- except Exception:- pass-- if resource_lock.active:- if not should_summarize and temp_uri and not source_committed:- viking_fs = get_viking_fs()- await viking_fs.persist_temp_tree(temp_uri, root_uri, ctx=ctx)- await rewrite_image_uris(root_uri, ctx=ctx, lock_handle=resource_lock.handle)- await viking_fs.delete_temp(parse_result.temp_dir_path, ctx=ctx)- await resource_lock.close()+ prepared = {+ "root_uri": root_uri,+ "temp_uri": temp_uri or parse_result.temp_dir_path,+ "temp_dir_path": parse_result.temp_dir_path,+ "source_committed": source_committed,+ "target_preexisting": target_preexisting,+ "is_code_repo": parse_result.source_format == "repository",+ }+ if defer_post_processing:+ result["_post_process"] = prepared+ result["_resource_lock"] = resource_lock+ else:+ post_result = await self.finish_prepared_resource(+ prepared,+ ctx=ctx,+ resource_lock=resource_lock,+ summarize=summarize,+ **kwargs,+ )+ if post_result.get("warnings"):+ result.setdefault("warnings", []).extend(post_result["warnings"])# 恢复原始 temp_uri 用于输出if original_temp_uri is not None:@@ -417,6 +389,80 @@ class ResourceProcessor:return result+ async def finish_prepared_resource(+ self,+ prepared: Dict[str, Any],+ *,+ ctx: RequestContext,+ resource_lock: LockLease = NO_LOCK,+ summarize: bool = False,+ **kwargs: Any,+ ) -> Dict[str, Any]:+ """Run the queue-producing phase for a resource already stored in VikingFS."""+ from openviking.metrics.datasources.resource import ResourceIngestionEventDataSource++ root_uri = str(prepared.get("root_uri") or "")+ temp_uri = prepared.get("temp_uri")+ temp_dir_path = prepared.get("temp_dir_path")+ source_committed = bool(prepared.get("source_committed"))+ target_preexisting = bool(prepared.get("target_preexisting"))+ build_index = bool(kwargs.get("build_index", True))+ should_summarize = summarize or build_index+ result: Dict[str, Any] = {"status": "success", "root_uri": root_uri}++ if should_summarize:+ stage_start = time.perf_counter()+ stage_status = "ok"+ try:+ with get_current_telemetry().measure("resource.summarize"):+ summary_result = await self._get_summarizer().summarize(+ resource_uris=[root_uri],+ ctx=ctx,+ skip_vectorization=not build_index,+ lock=resource_lock,+ temp_uris=[temp_uri],+ is_code_repo=bool(prepared.get("is_code_repo")),+ target_preexisting=target_preexisting,+ **kwargs,+ )+ if (+ resource_lock.active+ and summary_result.get("status") == "success"+ and summary_result.get("enqueued_count", 0) > 0+ ):+ await resource_lock.handoff()+ resource_lock = NO_LOCK+ except Exception as exc:+ logger.error("Summarization failed: %s", exc)+ result["warnings"] = [f"Summarization failed: {exc}"]+ stage_status = "error"+ finally:+ try:+ ResourceIngestionEventDataSource.record_stage(+ stage="summarize",+ status=stage_status,+ duration_seconds=float(time.perf_counter() - stage_start),+ account_id=getattr(ctx, "account_id", None),+ )+ except Exception:+ pass++ if resource_lock.active:+ try:+ if not should_summarize and temp_uri and not source_committed:+ viking_fs = get_viking_fs()+ await viking_fs.persist_temp_tree(temp_uri, root_uri, ctx=ctx)+ await rewrite_image_uris(+ root_uri,+ ctx=ctx,+ lock_handle=resource_lock.handle,+ )+ if temp_dir_path:+ await viking_fs.delete_temp(temp_dir_path, ctx=ctx)+ finally:+ await resource_lock.close()+ return result+async def reserve_unique_candidate(self,*,diff --git a/tests/storage/test_semantic_processor_target_preexisting.py b/tests/storage/test_semantic_processor_target_preexisting.pyindex 2546509a3..109d33a55 100644--- a/tests/storage/test_semantic_processor_target_preexisting.py+++ b/tests/storage/test_semantic_processor_target_preexisting.py@@ -149,3 +149,23 @@ async def test_sync_diff_reports_target_uris_and_preserves_sidecars(monkeypatch))assert fake_fs.contents["viking://resources/root/.abstract.md"] == "old abstract"assert fake_fs.deleted_temp == ["viking://temp/import"]+++async def test_sync_missing_source_never_touches_target(monkeypatch):+ fake_fs = AsyncMock()+ fake_fs.exists.return_value = False+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_processor.get_viking_fs",+ lambda: fake_fs,+ )++ with pytest.raises(FileNotFoundError, match="refusing to sync"):+ await SemanticProcessor()._sync_topdown_recursive(+ "viking://temp/missing",+ "viking://resources/root",+ lock=NO_LOCK,+ )++ fake_fs.ls.assert_not_awaited()+ fake_fs.rm.assert_not_awaited()