Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Feat/understand api queue (#2889)
chenxiaobin-lang61a34e555d3e
diff --git a/openviking/service/core.py b/openviking/service/core.pyindex da57163b3..827bc8bbb 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -27,6 +27,7 @@ from openviking.storage import VikingDBManagerfrom openviking.storage.collection_schemas import init_context_collectionfrom openviking.storage.index_consistency import check_index_consistencyfrom openviking.storage.queuefs.queue_manager import QueueManager, init_queue_manager+from openviking.storage.queuefs.understanding_parse_processor import UnderstandingParseProcessorfrom openviking.storage.transaction import LockManager, init_lock_managerfrom openviking.storage.viking_fs import VikingFS, init_viking_fsfrom openviking.utils.agfs_utils import (@@ -330,13 +331,9 @@ class OpenVikingService:if enable_recorder:logger.info("VikingFS IO Recorder enabled")- # Start queue workers now that VikingFS is ready.- # Doing it here (rather than in _init_storage) ensures that any tasks- # recovered from a previous crash are not processed before VikingFS is- # initialized, which would cause "VikingFS not initialized" errors.- if self._queue_manager:- self._queue_manager.start()- logger.info("QueueManager workers started")+ self._resource_processor = ResourceProcessor(+ vikingdb=self._vikingdb_manager,+ )# Initialize directoriesdirectory_initializer = DirectoryInitializer(@@ -356,9 +353,6 @@ class OpenVikingService:self._privacy_config_service = UserPrivacyConfigService(self._viking_fs)# Initialize processors- self._resource_processor = ResourceProcessor(- vikingdb=self._vikingdb_manager,- )self._skill_processor = SkillProcessor(vikingdb=self._vikingdb_manager,privacy_config_service=self._privacy_config_service,@@ -387,11 +381,6 @@ class OpenVikingService:privacy_config_service=self._privacy_config_service,resource_memory_link_service=self._resource_memory_link_service,)- self._resource_memory_link_service.set_dependencies(- vikingdb=self._vikingdb_manager,- viking_fs=self._viking_fs,- session_service=self._session_service,- )self._relation_service.set_viking_fs(self._viking_fs)self._pack_service.set_dependencies(viking_fs=self._viking_fs,@@ -411,12 +400,30 @@ class OpenVikingService:viking_fs=self._viking_fs,session_compressor=self._session_compressor,)+ self._resource_memory_link_service.set_dependencies(+ vikingdb=self._vikingdb_manager,+ viking_fs=self._viking_fs,+ session_service=self._session_service,+ )self._debug_service.set_dependencies(vikingdb=self._vikingdb_manager,config=self._config,agfs_client=self._agfs_client,)+ if self._queue_manager:+ external_parse_processor = UnderstandingParseProcessor(+ self._resource_processor,+ resource_memory_link_service=self._resource_memory_link_service,+ )+ self._queue_manager.get_queue(+ self._queue_manager.EXTERNAL_PARSE,+ dequeue_handler=external_parse_processor,+ allow_create=True,+ )+ self._queue_manager.start()+ logger.info("QueueManager workers started")+# Register as the process-wide service so flows that resolve the# service via the dependency global (e.g. background reindex tasks# triggered by git restore) work in embedded mode, not just under thediff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.pyindex 1dba7ba6f..f71503a5e 100644--- a/openviking/service/resource_service.py+++ b/openviking/service/resource_service.py@@ -155,6 +155,14 @@ class ResourceService:return Nonereturn self._watch_scheduler.watch_manager+ def _get_parser_router(self):+ if not hasattr(self, "_parser_router"):+ from openviking.parse.parser_router import ParserRouter+ from openviking.parse.registry import get_registry++ self._parser_router = ParserRouter(get_registry())+ return self._parser_router+def _sanitize_watch_processor_kwargs(self, processor_kwargs: Dict[str, Any]) -> Dict[str, Any]:sanitized: Dict[str, Any] = {}for key, value in processor_kwargs.items():@@ -165,6 +173,70 @@ class ResourceService:sanitized[key] = valuereturn sanitized+ async def _manage_watch_if_needed(+ self,+ *,+ watch_manager: Optional["WatchManager"],+ skip_watch_management: bool,+ watch_interval: float,+ target: ContentTargetSpec,+ root_uri: str,+ path: str,+ reason: str,+ instruction: str,+ build_index: bool,+ summarize: bool,+ processor_kwargs: Dict[str, Any],+ watch_auth_state: Optional[Dict[str, Any]],+ ctx: RequestContext,+ ) -> None:+ if not watch_manager or skip_watch_management:+ return+ telemetry = get_current_telemetry()+ with telemetry.measure("resource.watch"):+ if watch_interval > 0:+ watch_to = target.to+ parent_uri = target.parent+ if not watch_to:+ watch_to = validate_optional_content_target_uri(+ root_uri,+ ctx,+ kind="resource",+ field_name="root_uri",+ )+ parent_uri = None+ if not watch_to:+ raise InvalidArgumentError(+ "watch_interval > 0 requires a stable target URI. "+ "Pass 'to' explicitly, or add a resource type that returns root_uri."+ )+ try:+ sanitized = self._sanitize_watch_processor_kwargs(processor_kwargs)+ if watch_auth_state is not None:+ sanitized.pop(FEISHU_ACCESS_TOKEN_ARG, None)+ await self._handle_watch_task_creation(+ path=path,+ to_uri=watch_to,+ parent_uri=parent_uri,+ reason=reason,+ instruction=instruction,+ watch_interval=watch_interval,+ build_index=build_index,+ summarize=summarize,+ processor_kwargs=sanitized,+ auth_state=watch_auth_state,+ ctx=ctx,+ )+ except ConflictError:+ raise+ except Exception as e:+ logger.warning(f"[ResourceService] Failed to create watch task for {watch_to}: {e}")+ elif target.to:+ try:+ await self._handle_watch_task_cancellation(to_uri=target.to, ctx=ctx)+ except Exception as e:+ logger.warning(f"[ResourceService] Failed to cancel watch task for {target.to}: {e}")+def _normalize_add_resource_args(self,args: Optional[Dict[str, Any]],@@ -458,6 +530,9 @@ class ResourceService:)return root_uri, resource_lock+ def _should_use_understanding_api(self, path: str) -> bool:+ return self._get_parser_router().should_use_understanding_api(path)+@staticmethoddef _target_doc_name(path: str,@@ -626,6 +701,163 @@ class ResourceService:if resource_lock is not None:kwargs["resource_lock"] = resource_lock+ if (+ not wait+ and not is_git_repo_url(path)+ and self._should_use_understanding_api(path)+ and not allow_local_path_resolution+ and self._resource_processor is not None+ ):+ from openviking.service.task_tracker import get_task_tracker+ from openviking.storage.queuefs import QueueManager, get_queue_manager+ from openviking.storage.queuefs.understanding_parse_msg import (+ UnderstandingParseMsg,+ )++ source_name = kwargs.get("source_name")+ source_info = _ResourceSourceInfo(+ source_name=source_name,+ source_path=path,+ source_format="file",+ )+ doc_name = self._target_doc_name(path, source_name, source_info)+ source_path = source_info.source_path or source_name or path+ root_uri, candidate_uri = await self._resource_processor.tree_builder.resolve_target_uri(+ ctx=ctx,+ doc_name=doc_name,+ scope="resources",+ to_uri=target.to,+ parent_uri=target.parent,+ source_path=source_path,+ source_format=source_info.source_format,+ create_parent=target.create_parent,+ )+ if self._viking_fs is None:+ raise NotInitializedError("VikingFS")+ from openviking.storage.errors import LockAcquisitionError, ResourceBusyError+ from openviking.storage.transaction import OwnedLockLease, get_lock_manager++ lock_manager = get_lock_manager()+ lock_lease: LockLease = NO_LOCK++ async def _reserve_tree(uri: str) -> LockLease:+ dst_path = self._viking_fs._uri_to_path(uri, ctx=ctx)+ try:+ return await OwnedLockLease.acquire_tree(lock_manager, dst_path, timeout=0.0)+ except LockAcquisitionError as exc:+ raise ResourceBusyError(+ f"Resource is busy: {uri}",+ uri=uri,+ conflict_type="path_busy",+ retryable=True,+ ) from exc++ if candidate_uri:+ max_attempts = 100+ reserved = False+ for attempt in range(max_attempts + 1):+ attempt_uri = candidate_uri if attempt == 0 else f"{candidate_uri}_{attempt}"+ if await self._viking_fs.exists(attempt_uri, ctx=ctx):+ continue+ try:+ lock_lease = await _reserve_tree(attempt_uri)+ root_uri = attempt_uri+ reserved = True+ break+ except ResourceBusyError:+ continue+ if not reserved:+ raise FileExistsError(+ f"Cannot resolve unique name for {candidate_uri} after {max_attempts} attempts"+ )+ else:+ lock_lease = await _reserve_tree(root_uri)++ task_tracker = get_task_tracker()+ task = None+ try:+ task = await task_tracker.create(+ "add_resource",+ resource_id=root_uri,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ await task_tracker.update_stage(+ task.task_id,+ "queued_external_parse",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )++ lock_handoff = lock_lease.to_handoff()+ msg = UnderstandingParseMsg(+ task_id=task.task_id,+ telemetry_id=telemetry_id or None,+ path=path,+ root_uri=root_uri,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ role=str(ctx.role),+ actor_peer_id=ctx.actor_peer_id,+ reason=reason,+ instruction=instruction,+ build_index=build_index,+ summarize=summarize,+ strict=bool(kwargs.get("strict", False)),+ ignore_dirs=kwargs.get("ignore_dirs"),+ include=kwargs.get("include"),+ exclude=kwargs.get("exclude"),+ directly_upload_media=bool(kwargs.get("directly_upload_media", True)),+ allow_local_path_resolution=allow_local_path_resolution,+ enforce_public_remote_targets=enforce_public_remote_targets,+ args=normalized_args.processor_kwargs,+ source_name=source_name,+ lock_handoff=lock_handoff.to_dict() if lock_handoff else None,+ )+ qm = get_queue_manager()+ await qm.enqueue(QueueManager.EXTERNAL_PARSE, msg.to_dict())+ await lock_lease.handoff()+ monitor_started = True+ except Exception as exc:+ with contextlib.suppress(Exception):+ await lock_lease.close()++ if task is not None:+ with contextlib.suppress(Exception):+ await task_tracker.fail(+ task.task_id,+ str(exc),+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )++ raise+ logger.info(+ "[ResourceService] Enqueued UnderstandingParseMsg task_id=%s root_uri=%s",+ task.task_id,+ root_uri,+ )+ await self._manage_watch_if_needed(+ watch_manager=watch_manager,+ skip_watch_management=skip_watch_management,+ watch_interval=watch_interval,+ target=target,+ root_uri=root_uri,+ path=path,+ reason=reason,+ instruction=instruction,+ build_index=build_index,+ summarize=summarize,+ processor_kwargs=kwargs,+ watch_auth_state=normalized_args.watch_auth_state,+ ctx=ctx,+ )+ return {+ "status": "success",+ "root_uri": root_uri,+ "task_id": task.task_id,+ }+result = await self._resource_processor.process_resource(path=path,ctx=ctx,@@ -690,54 +922,21 @@ class ResourceService:root_uri=result.get("root_uri"),)telemetry.set("queue.wait.duration_ms", queue_wait_duration_ms)- if watch_manager and not skip_watch_management:- with telemetry.measure("resource.watch"):- if watch_interval > 0:- watch_to = target.to- parent_uri = target.parent- if not watch_to:- watch_to = validate_optional_content_target_uri(- result.get("root_uri"),- ctx,- kind="resource",- field_name="root_uri",- )- parent_uri = None- if not watch_to:- raise InvalidArgumentError(- "watch_interval > 0 requires a stable target URI. "- "Pass 'to' explicitly, or add a resource type that returns root_uri."- )- try:- processor_kwargs = self._sanitize_watch_processor_kwargs(kwargs)- if normalized_args.watch_auth_state is not None:- processor_kwargs.pop(FEISHU_ACCESS_TOKEN_ARG, None)- await self._handle_watch_task_creation(- path=path,- to_uri=watch_to,- parent_uri=parent_uri,- reason=reason,- instruction=instruction,- watch_interval=watch_interval,- build_index=build_index,- summarize=summarize,- processor_kwargs=processor_kwargs,- auth_state=normalized_args.watch_auth_state,- ctx=ctx,- )- except ConflictError:- raise- except Exception as e:- logger.warning(- f"[ResourceService] Failed to create watch task for {watch_to}: {e}"- )- elif target.to:- try:- await self._handle_watch_task_cancellation(to_uri=target.to, ctx=ctx)- except Exception as e:- logger.warning(- f"[ResourceService] Failed to cancel watch task for {target.to}: {e}"- )+ await self._manage_watch_if_needed(+ watch_manager=watch_manager,+ skip_watch_management=skip_watch_management,+ watch_interval=watch_interval,+ target=target,+ root_uri=str(result.get("root_uri") or ""),+ path=path,+ reason=reason,+ instruction=instruction,+ build_index=build_index,+ summarize=summarize,+ processor_kwargs=kwargs,+ watch_auth_state=normalized_args.watch_auth_state,+ ctx=ctx,+ )if wait:await self._link_resource_reason_memory(result=result,diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 962e62291..3c40b5b58 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -30,6 +30,7 @@ def init_queue_manager(mount_point: str = "/queue",max_concurrent_embedding: int = 10,max_concurrent_semantic: int = 64,+ max_concurrent_external_parse: int = 4,) -> "QueueManager":"""Initialize QueueManager singleton.@@ -47,6 +48,7 @@ def init_queue_manager(mount_point=mount_point,max_concurrent_embedding=max_concurrent_embedding,max_concurrent_semantic=max_concurrent_semantic,+ max_concurrent_external_parse=max_concurrent_external_parse,)return _instance@@ -67,6 +69,7 @@ class QueueManager:# Standard queue namesEMBEDDING = "Embedding"SEMANTIC = "Semantic"+ EXTERNAL_PARSE = "ExternalParse"def __init__(self,@@ -75,6 +78,7 @@ class QueueManager:mount_point: str = "/queue",max_concurrent_embedding: int = 10,max_concurrent_semantic: int = 64,+ max_concurrent_external_parse: int = 4,):"""Initialize QueueManager."""self._agfs = agfs@@ -82,6 +86,7 @@ class QueueManager:self.mount_point = mount_pointself._max_concurrent_embedding = max_concurrent_embeddingself._max_concurrent_semantic = max_concurrent_semantic+ self._max_concurrent_external_parse = max_concurrent_external_parseself._queues: Dict[str, NamedQueue] = {}self._started = Falseself._queue_threads: Dict[str, threading.Thread] = {}@@ -149,11 +154,12 @@ class QueueManager:if thread.is_alive():return- max_concurrent = (- self._max_concurrent_embedding- if queue.name == self.EMBEDDING- else self._max_concurrent_semantic- )+ if queue.name == self.EMBEDDING:+ max_concurrent = self._max_concurrent_embedding+ elif queue.name == self.EXTERNAL_PARSE:+ max_concurrent = self._max_concurrent_external_parse+ else:+ max_concurrent = self._max_concurrent_semanticstop_event = threading.Event()self._queue_stop_events[queue.name] = stop_eventthread = threading.Thread(diff --git a/openviking/storage/queuefs/understanding_parse_msg.py b/openviking/storage/queuefs/understanding_parse_msg.pynew file mode 100644index 000000000..12ea5c731--- /dev/null+++ b/openviking/storage/queuefs/understanding_parse_msg.py@@ -0,0 +1,155 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""UnderstandingParseMsg: External parse queue message for UnderstandingAPI."""++import json+from dataclasses import asdict, dataclass, field+from datetime import datetime+from typing import Any, Dict, Optional+from uuid import uuid4+++@dataclass+class UnderstandingParseMsg:+ id: str+ task_id: str+ telemetry_id: Optional[str]+ path: str+ root_uri: str+ lock_handoff: Optional[Dict[str, Any]] = None+ status: str = "pending"+ timestamp: int = int(datetime.now().timestamp())+ account_id: str = "default"+ user_id: str = "default"+ role: str = "root"+ actor_peer_id: Optional[str] = None+ reason: str = ""+ instruction: str = ""+ build_index: bool = True+ summarize: bool = False+ strict: bool = False+ ignore_dirs: Optional[str] = None+ include: Optional[str] = None+ exclude: Optional[str] = None+ directly_upload_media: bool = True+ allow_local_path_resolution: bool = True+ enforce_public_remote_targets: bool = False+ args: Dict[str, Any] = field(default_factory=dict)+ source_name: Optional[str] = None++ def __init__(+ self,+ *,+ task_id: str,+ path: str,+ root_uri: str,+ account_id: str,+ user_id: str,+ role: str,+ actor_peer_id: Optional[str] = None,+ telemetry_id: Optional[str] = None,+ lock_handoff: Optional[Dict[str, Any]] = None,+ reason: str = "",+ instruction: str = "",+ build_index: bool = True,+ summarize: bool = False,+ strict: bool = False,+ ignore_dirs: Optional[str] = None,+ include: Optional[str] = None,+ exclude: Optional[str] = None,+ directly_upload_media: bool = True,+ allow_local_path_resolution: bool = True,+ enforce_public_remote_targets: bool = False,+ args: Optional[Dict[str, Any]] = None,+ source_name: Optional[str] = None,+ ):+ self.id = str(uuid4())+ self.task_id = task_id+ self.telemetry_id = telemetry_id+ self.path = path+ self.root_uri = root_uri+ self.account_id = account_id+ self.user_id = user_id+ self.role = role+ self.actor_peer_id = actor_peer_id+ self.lock_handoff = lock_handoff+ self.reason = reason+ self.instruction = instruction+ self.build_index = build_index+ self.summarize = summarize+ self.strict = strict+ self.ignore_dirs = ignore_dirs+ self.include = include+ self.exclude = exclude+ self.directly_upload_media = directly_upload_media+ self.allow_local_path_resolution = allow_local_path_resolution+ self.enforce_public_remote_targets = enforce_public_remote_targets+ self.args = args or {}+ self.source_name = source_name++ def to_dict(self) -> Dict[str, Any]:+ return asdict(self)++ def to_json(self) -> str:+ return json.dumps(self.to_dict())++ @classmethod+ def from_dict(cls, data: Dict[str, Any]) -> "UnderstandingParseMsg":+ if not data:+ raise ValueError("Data dictionary is empty")+ task_id = data.get("task_id")+ path = data.get("path")+ root_uri = data.get("root_uri")+ if not task_id or not path or not root_uri:+ missing = []+ if not task_id:+ missing.append("task_id")+ if not path:+ missing.append("path")+ if not root_uri:+ missing.append("root_uri")+ raise ValueError(f"Missing required fields: {missing}")++ obj = cls(+ task_id=str(task_id),+ path=str(path),+ root_uri=str(root_uri),+ account_id=str(data.get("account_id", "default")),+ user_id=str(data.get("user_id", "default")),+ role=str(data.get("role", "root")),+ actor_peer_id=data.get("actor_peer_id"),+ telemetry_id=str(data.get("telemetry_id"))+ if isinstance(data.get("telemetry_id"), str)+ else None,+ lock_handoff=data.get("lock_handoff")+ if isinstance(data.get("lock_handoff"), dict)+ else None,+ reason=str(data.get("reason", "")),+ instruction=str(data.get("instruction", "")),+ build_index=bool(data.get("build_index", True)),+ summarize=bool(data.get("summarize", False)),+ strict=bool(data.get("strict", False)),+ ignore_dirs=data.get("ignore_dirs"),+ include=data.get("include"),+ exclude=data.get("exclude"),+ directly_upload_media=bool(data.get("directly_upload_media", True)),+ allow_local_path_resolution=bool(data.get("allow_local_path_resolution", True)),+ enforce_public_remote_targets=bool(data.get("enforce_public_remote_targets", False)),+ args=data.get("args") if isinstance(data.get("args"), dict) else {},+ source_name=data.get("source_name"),+ )+ if data.get("id"):+ obj.id = str(data["id"])+ if "status" in data:+ obj.status = str(data["status"])+ if "timestamp" in data:+ obj.timestamp = int(data["timestamp"])+ return obj++ @classmethod+ def from_json(cls, json_str: str) -> "UnderstandingParseMsg":+ try:+ data = json.loads(json_str)+ except json.JSONDecodeError as e:+ raise ValueError(f"Invalid JSON string: {e}")+ return cls.from_dict(data)diff --git a/openviking/storage/queuefs/understanding_parse_processor.py b/openviking/storage/queuefs/understanding_parse_processor.pynew file mode 100644index 000000000..38f6e770b--- /dev/null+++ b/openviking/storage/queuefs/understanding_parse_processor.py@@ -0,0 +1,344 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""UnderstandingParseProcessor: Processes UnderstandingParseMsg messages."""++import asyncio+import json++from contextlib import suppress+from typing import Any, Dict, Optional++from openviking.observability.context import (+ bind_execution_context,+)+from openviking.server.identity import RequestContext, Role+from openviking.service.task_tracker import get_task_tracker+from openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.understanding_parse_msg import UnderstandingParseMsg+from openviking.telemetry import bind_telemetry, bind_telemetry_stage, resolve_telemetry+from openviking_cli.session.user_id import UserIdentifier+from openviking_cli.utils.logger import get_logger++logger = get_logger(__name__)+++class UnderstandingParseProcessor(DequeueHandlerBase):+ def __init__(self, resource_processor: Any, resource_memory_link_service: Any = None):+ self._resource_processor = resource_processor+ self._resource_memory_link_service = resource_memory_link_service+ self._background_tasks: set[asyncio.Task] = set()++ async def _monitor_queue_then_link_memory(+ self,+ *,+ task_id: str,+ telemetry_id: str,+ ctx: RequestContext,+ result: Dict[str, Any],+ reason: str,+ source_name: Optional[str],+ ) -> None:+ from openviking.telemetry.resource_summary import unregister_wait_telemetry+ from openviking.telemetry.request_wait_tracker import get_request_wait_tracker++ task_tracker = get_task_tracker()+ request_wait_tracker = get_request_wait_tracker()+ try:+ request_wait_tracker.register_request(telemetry_id)+ await request_wait_tracker.wait_for_request(telemetry_id)+ status = request_wait_tracker.build_queue_status(telemetry_id)+ errors = sum(int(group.get("error_count", 0) or 0) for group in status.values())+ if errors:+ await task_tracker.fail(+ task_id,+ f"queue processing failed: {status}",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_error("queue processing failed", {"task_id": task_id})+ return++ result["queue_status"] = status+ if self._resource_memory_link_service and (reason or "").strip():+ root_uri = result.get("root_uri")+ if root_uri:+ try:+ link_result = await self._resource_memory_link_service.on_resource_added(+ ctx=ctx,+ resource_uri=root_uri,+ reason=reason,+ source_name=source_name,+ timeout=None,+ )+ result["memory_linking"] = link_result+ except Exception as exc:+ logger.warning(+ "[UnderstandingParse] Failed to link resource reason memory: %s",+ exc,+ )+ result.setdefault("warnings", []).append(f"Memory linking failed: {exc}")++ await task_tracker.complete(+ task_id,+ result,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_success()+ except Exception as exc:+ await task_tracker.fail(+ task_id,+ str(exc),+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_error(str(exc), {"task_id": task_id})+ finally:+ request_wait_tracker.cleanup(telemetry_id)+ unregister_wait_telemetry(telemetry_id)++ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:+ if not data:+ return None++ payload: Dict[str, Any] = data+ if isinstance(data, dict) and "data" in data:+ inner = data.get("data")+ if isinstance(inner, str) and inner.strip():+ try:+ decoded = json.loads(inner)+ if isinstance(decoded, dict):+ payload = decoded+ except Exception:+ payload = data+ elif isinstance(inner, dict):+ payload = inner+ if isinstance(payload, dict) and data.get("id") and not payload.get("id"):+ payload["id"] = data["id"]++ msg = UnderstandingParseMsg.from_dict(payload)+ ctx = RequestContext(+ user=UserIdentifier(msg.account_id, msg.user_id),+ role=Role(msg.role),+ actor_peer_id=msg.actor_peer_id,+ )++ task_tracker = get_task_tracker()+ lock_lease = None+ if msg.lock_handoff is not None:+ try:+ from openviking.storage.transaction.lock_lease import (+ LockHandoffRef,+ OwnedLockLease,+ )++ ref = LockHandoffRef.from_value(msg.lock_handoff)+ if ref is None:+ raise ValueError("Invalid lock_handoff")+ lock_lease = await OwnedLockLease.from_handoff(ref)+ except Exception as exc:+ retry_key = "_lock_handoff_retry"+ raw_retry = (msg.args or {}).get(retry_key, 0)+ try:+ retry_count = int(raw_retry or 0)+ except (TypeError, ValueError):+ retry_count = 0++ max_retries = 2+ if retry_count < max_retries:+ from openviking.storage.queuefs import QueueManager, get_queue_manager++ qm = get_queue_manager()+ if qm is None:+ raise+ new_args = dict(msg.args or {})+ new_args[retry_key] = retry_count + 1+ retry_msg = UnderstandingParseMsg(+ task_id=msg.task_id,+ telemetry_id=msg.telemetry_id,+ path=msg.path,+ root_uri=msg.root_uri,+ account_id=msg.account_id,+ user_id=msg.user_id,+ role=msg.role,+ actor_peer_id=msg.actor_peer_id,+ lock_handoff=msg.lock_handoff,+ reason=msg.reason,+ instruction=msg.instruction,+ build_index=msg.build_index,+ summarize=msg.summarize,+ strict=msg.strict,+ ignore_dirs=msg.ignore_dirs,+ include=msg.include,+ exclude=msg.exclude,+ directly_upload_media=msg.directly_upload_media,+ allow_local_path_resolution=msg.allow_local_path_resolution,+ enforce_public_remote_targets=msg.enforce_public_remote_targets,+ args=new_args,+ source_name=msg.source_name,+ )+ await qm.enqueue(QueueManager.EXTERNAL_PARSE, retry_msg.to_dict())+ self.report_requeue()+ self.report_success()+ return None++ await task_tracker.fail(+ msg.task_id,+ f"Invalid lock_handoff: {exc}",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_error(f"Invalid lock_handoff: {exc}", data)+ return None++ logger.info(+ "[UnderstandingParse] Dequeued task_id=%s root_uri=%s",+ msg.task_id,+ msg.root_uri,+ )+ resource_processor = self._resource_processor+ if resource_processor is None:+ raise RuntimeError("ResourceProcessor is None")++ telemetry_id = msg.telemetry_id or ""+ if telemetry_id:+ from openviking.telemetry.request_wait_tracker import get_request_wait_tracker++ get_request_wait_tracker().register_request(telemetry_id)++ telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None+ if telemetry_id and telemetry is None:+ from openviking.telemetry.operation import OperationTelemetry++ telemetry = OperationTelemetry(operation="noop", enabled=False)+ telemetry.telemetry_id = telemetry_id++ with bind_execution_context(), (bind_telemetry(telemetry) if telemetry else suppress()):+ try:+ await task_tracker.start(+ msg.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id+ )+ await task_tracker.update_stage(+ msg.task_id,+ "parsing",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )++ bind_telemetry_stage("parse_understanding")+ kwargs: Dict[str, Any] = {}+ if lock_lease is not None and getattr(lock_lease, "active", False):+ kwargs["resource_lock"] = lock_lease+ if msg.enforce_public_remote_targets:+ from openviking.utils.network_guard import ensure_public_remote_target++ kwargs.setdefault("request_validator", ensure_public_remote_target)+ result = await resource_processor.process_resource(+ path=msg.path,+ ctx=ctx,+ reason=msg.reason,+ instruction=msg.instruction,+ scope="resources",+ to=msg.root_uri,+ parent=None,+ build_index=msg.build_index,+ summarize=msg.summarize,+ allow_local_path_resolution=msg.allow_local_path_resolution,+ enforce_public_remote_targets=msg.enforce_public_remote_targets,+ strict=msg.strict,+ source_name=msg.source_name,+ ignore_dirs=msg.ignore_dirs,+ include=msg.include,+ exclude=msg.exclude,+ directly_upload_media=msg.directly_upload_media,+ args=msg.args,+ skip_watch_management=True,+ watch_interval=0,+ **kwargs,+ )++ if result.get("status") == "error":+ errors = result.get("errors") or ["resource processing failed"]+ await task_tracker.fail(+ msg.task_id,+ "; ".join(str(error) for error in errors),+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_error("resource processing failed", data)+ return None++ if telemetry_id:+ await task_tracker.update_stage(+ msg.task_id,+ "processing_queue",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ monitor = asyncio.create_task(+ self._monitor_queue_then_link_memory(+ task_id=msg.task_id,+ telemetry_id=telemetry_id,+ ctx=ctx,+ result=result,+ reason=msg.reason,+ source_name=msg.source_name,+ )+ )+ self._background_tasks.add(monitor)+ monitor.add_done_callback(self._background_tasks.discard)+ return None++ if self._resource_memory_link_service and (msg.reason or "").strip():+ root_uri = result.get("root_uri")+ if root_uri:+ try:+ link_result = await self._resource_memory_link_service.on_resource_added(+ ctx=ctx,+ resource_uri=root_uri,+ reason=msg.reason,+ source_name=msg.source_name,+ timeout=None,+ )+ result["memory_linking"] = link_result+ except Exception as exc:+ logger.warning(+ "[UnderstandingParse] Failed to link resource reason memory: %s",+ exc,+ )+ result.setdefault("warnings", []).append(+ f"Memory linking failed: {exc}"+ )++ await task_tracker.complete(+ msg.task_id,+ result,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_success()+ return None+ except asyncio.CancelledError:+ await task_tracker.fail(+ msg.task_id,+ "external parse cancelled",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_error("external parse cancelled", data)+ raise+ except Exception as exc:+ await task_tracker.fail(+ msg.task_id,+ str(exc),+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ self.report_error(str(exc), data)+ return None+ finally:+ try:+ if lock_lease is not None:+ await lock_lease.close()+ except Exception:+ passdiff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex f2ebf9e79..ab759659e 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -23,7 +23,8 @@ from openviking.storage.transaction import (LockLease,OwnedLockLease,)-from openviking.storage.viking_fs import get_viking_fs+from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES+from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fsfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.embedding_utils import index_resourcefrom openviking.utils.summarizer import Summarizer@@ -298,6 +299,26 @@ class ResourceProcessor:result["root_uri"] = root_urielse:target_preexisting = await viking_fs.exists(root_uri, ctx=ctx)+ if target_preexisting:+ try:+ stat = await viking_fs.stat(root_uri, ctx=ctx)+ if isinstance(stat, dict) and stat.get("isDir"):+ entries = await viking_fs.ls(+ root_uri,+ show_all_hidden=True,+ node_limit=LS_ALL_NODES,+ ctx=ctx,+ )+ names: list[str] = []+ for entry in entries:+ name = entry.get("name", "")+ if not name or name in {".", ".."}:+ continue+ names.append(str(name))+ if all(name in STORAGE_INTERNAL_ENTRY_NAMES for name in names):+ target_preexisting = False+ except Exception:+ passif not resource_lock.active:dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)resource_lock = await self.acquire_resource_lock(