Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Feat/understand api queue (#2889)

chenxiaobin-lang61a34e555d3e
diff --git a/openviking/service/core.py b/openviking/service/core.py
index da57163b3..827bc8bbb 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -27,6 +27,7 @@ from openviking.storage import VikingDBManager
from openviking.storage.collection_schemas import init_context_collection
from openviking.storage.index_consistency import check_index_consistency
from openviking.storage.queuefs.queue_manager import QueueManager, init_queue_manager
+from openviking.storage.queuefs.understanding_parse_processor import UnderstandingParseProcessor
from openviking.storage.transaction import LockManager, init_lock_manager
from openviking.storage.viking_fs import VikingFS, init_viking_fs
from openviking.utils.agfs_utils import (
@@ -330,13 +331,9 @@ class OpenVikingService:
if enable_recorder:
logger.info("VikingFS IO Recorder enabled")
- # Start queue workers now that VikingFS is ready.
- # Doing it here (rather than in _init_storage) ensures that any tasks
- # recovered from a previous crash are not processed before VikingFS is
- # initialized, which would cause "VikingFS not initialized" errors.
- if self._queue_manager:
- self._queue_manager.start()
- logger.info("QueueManager workers started")
+ self._resource_processor = ResourceProcessor(
+ vikingdb=self._vikingdb_manager,
+ )
# Initialize directories
directory_initializer = DirectoryInitializer(
@@ -356,9 +353,6 @@ class OpenVikingService:
self._privacy_config_service = UserPrivacyConfigService(self._viking_fs)
# Initialize processors
- self._resource_processor = ResourceProcessor(
- vikingdb=self._vikingdb_manager,
- )
self._skill_processor = SkillProcessor(
vikingdb=self._vikingdb_manager,
privacy_config_service=self._privacy_config_service,
@@ -387,11 +381,6 @@ class OpenVikingService:
privacy_config_service=self._privacy_config_service,
resource_memory_link_service=self._resource_memory_link_service,
)
- self._resource_memory_link_service.set_dependencies(
- vikingdb=self._vikingdb_manager,
- viking_fs=self._viking_fs,
- session_service=self._session_service,
- )
self._relation_service.set_viking_fs(self._viking_fs)
self._pack_service.set_dependencies(
viking_fs=self._viking_fs,
@@ -411,12 +400,30 @@ class OpenVikingService:
viking_fs=self._viking_fs,
session_compressor=self._session_compressor,
)
+ self._resource_memory_link_service.set_dependencies(
+ vikingdb=self._vikingdb_manager,
+ viking_fs=self._viking_fs,
+ session_service=self._session_service,
+ )
self._debug_service.set_dependencies(
vikingdb=self._vikingdb_manager,
config=self._config,
agfs_client=self._agfs_client,
)
+ if self._queue_manager:
+ external_parse_processor = UnderstandingParseProcessor(
+ self._resource_processor,
+ resource_memory_link_service=self._resource_memory_link_service,
+ )
+ self._queue_manager.get_queue(
+ self._queue_manager.EXTERNAL_PARSE,
+ dequeue_handler=external_parse_processor,
+ allow_create=True,
+ )
+ self._queue_manager.start()
+ logger.info("QueueManager workers started")
+
# Register as the process-wide service so flows that resolve the
# service via the dependency global (e.g. background reindex tasks
# triggered by git restore) work in embedded mode, not just under the
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.py
index 1dba7ba6f..f71503a5e 100644
--- a/openviking/service/resource_service.py
+++ b/openviking/service/resource_service.py
@@ -155,6 +155,14 @@ class ResourceService:
return None
return self._watch_scheduler.watch_manager
+ def _get_parser_router(self):
+ if not hasattr(self, "_parser_router"):
+ from openviking.parse.parser_router import ParserRouter
+ from openviking.parse.registry import get_registry
+
+ self._parser_router = ParserRouter(get_registry())
+ return self._parser_router
+
def _sanitize_watch_processor_kwargs(self, processor_kwargs: Dict[str, Any]) -> Dict[str, Any]:
sanitized: Dict[str, Any] = {}
for key, value in processor_kwargs.items():
@@ -165,6 +173,70 @@ class ResourceService:
sanitized[key] = value
return sanitized
+ async def _manage_watch_if_needed(
+ self,
+ *,
+ watch_manager: Optional["WatchManager"],
+ skip_watch_management: bool,
+ watch_interval: float,
+ target: ContentTargetSpec,
+ root_uri: str,
+ path: str,
+ reason: str,
+ instruction: str,
+ build_index: bool,
+ summarize: bool,
+ processor_kwargs: Dict[str, Any],
+ watch_auth_state: Optional[Dict[str, Any]],
+ ctx: RequestContext,
+ ) -> None:
+ if not watch_manager or skip_watch_management:
+ return
+ telemetry = get_current_telemetry()
+ with telemetry.measure("resource.watch"):
+ if watch_interval > 0:
+ watch_to = target.to
+ parent_uri = target.parent
+ if not watch_to:
+ watch_to = validate_optional_content_target_uri(
+ root_uri,
+ ctx,
+ kind="resource",
+ field_name="root_uri",
+ )
+ parent_uri = None
+ if not watch_to:
+ raise InvalidArgumentError(
+ "watch_interval > 0 requires a stable target URI. "
+ "Pass 'to' explicitly, or add a resource type that returns root_uri."
+ )
+ try:
+ sanitized = self._sanitize_watch_processor_kwargs(processor_kwargs)
+ if watch_auth_state is not None:
+ sanitized.pop(FEISHU_ACCESS_TOKEN_ARG, None)
+ await self._handle_watch_task_creation(
+ path=path,
+ to_uri=watch_to,
+ parent_uri=parent_uri,
+ reason=reason,
+ instruction=instruction,
+ watch_interval=watch_interval,
+ build_index=build_index,
+ summarize=summarize,
+ processor_kwargs=sanitized,
+ auth_state=watch_auth_state,
+ ctx=ctx,
+ )
+ except ConflictError:
+ raise
+ except Exception as e:
+ logger.warning(f"[ResourceService] Failed to create watch task for {watch_to}: {e}")
+ elif target.to:
+ try:
+ await self._handle_watch_task_cancellation(to_uri=target.to, ctx=ctx)
+ except Exception as e:
+ logger.warning(f"[ResourceService] Failed to cancel watch task for {target.to}: {e}")
+
def _normalize_add_resource_args(
self,
args: Optional[Dict[str, Any]],
@@ -458,6 +530,9 @@ class ResourceService:
)
return root_uri, resource_lock
+ def _should_use_understanding_api(self, path: str) -> bool:
+ return self._get_parser_router().should_use_understanding_api(path)
+
@staticmethod
def _target_doc_name(
path: str,
@@ -626,6 +701,163 @@ class ResourceService:
if resource_lock is not None:
kwargs["resource_lock"] = resource_lock
+ if (
+ not wait
+ and not is_git_repo_url(path)
+ and self._should_use_understanding_api(path)
+ and not allow_local_path_resolution
+ and self._resource_processor is not None
+ ):
+ from openviking.service.task_tracker import get_task_tracker
+ from openviking.storage.queuefs import QueueManager, get_queue_manager
+ from openviking.storage.queuefs.understanding_parse_msg import (
+ UnderstandingParseMsg,
+ )
+
+ source_name = kwargs.get("source_name")
+ source_info = _ResourceSourceInfo(
+ source_name=source_name,
+ source_path=path,
+ source_format="file",
+ )
+ doc_name = self._target_doc_name(path, source_name, source_info)
+ source_path = source_info.source_path or source_name or path
+ root_uri, candidate_uri = await self._resource_processor.tree_builder.resolve_target_uri(
+ ctx=ctx,
+ doc_name=doc_name,
+ scope="resources",
+ to_uri=target.to,
+ parent_uri=target.parent,
+ source_path=source_path,
+ source_format=source_info.source_format,
+ create_parent=target.create_parent,
+ )
+ if self._viking_fs is None:
+ raise NotInitializedError("VikingFS")
+ from openviking.storage.errors import LockAcquisitionError, ResourceBusyError
+ from openviking.storage.transaction import OwnedLockLease, get_lock_manager
+
+ lock_manager = get_lock_manager()
+ lock_lease: LockLease = NO_LOCK
+
+ async def _reserve_tree(uri: str) -> LockLease:
+ dst_path = self._viking_fs._uri_to_path(uri, ctx=ctx)
+ try:
+ return await OwnedLockLease.acquire_tree(lock_manager, dst_path, timeout=0.0)
+ except LockAcquisitionError as exc:
+ raise ResourceBusyError(
+ f"Resource is busy: {uri}",
+ uri=uri,
+ conflict_type="path_busy",
+ retryable=True,
+ ) from exc
+
+ if candidate_uri:
+ max_attempts = 100
+ reserved = False
+ for attempt in range(max_attempts + 1):
+ attempt_uri = candidate_uri if attempt == 0 else f"{candidate_uri}_{attempt}"
+ if await self._viking_fs.exists(attempt_uri, ctx=ctx):
+ continue
+ try:
+ lock_lease = await _reserve_tree(attempt_uri)
+ root_uri = attempt_uri
+ reserved = True
+ break
+ except ResourceBusyError:
+ continue
+ if not reserved:
+ raise FileExistsError(
+ f"Cannot resolve unique name for {candidate_uri} after {max_attempts} attempts"
+ )
+ else:
+ lock_lease = await _reserve_tree(root_uri)
+
+ task_tracker = get_task_tracker()
+ task = None
+ try:
+ task = await task_tracker.create(
+ "add_resource",
+ resource_id=root_uri,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ await task_tracker.update_stage(
+ task.task_id,
+ "queued_external_parse",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+
+ lock_handoff = lock_lease.to_handoff()
+ msg = UnderstandingParseMsg(
+ task_id=task.task_id,
+ telemetry_id=telemetry_id or None,
+ path=path,
+ root_uri=root_uri,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ role=str(ctx.role),
+ actor_peer_id=ctx.actor_peer_id,
+ reason=reason,
+ instruction=instruction,
+ build_index=build_index,
+ summarize=summarize,
+ strict=bool(kwargs.get("strict", False)),
+ ignore_dirs=kwargs.get("ignore_dirs"),
+ include=kwargs.get("include"),
+ exclude=kwargs.get("exclude"),
+ directly_upload_media=bool(kwargs.get("directly_upload_media", True)),
+ allow_local_path_resolution=allow_local_path_resolution,
+ enforce_public_remote_targets=enforce_public_remote_targets,
+ args=normalized_args.processor_kwargs,
+ source_name=source_name,
+ lock_handoff=lock_handoff.to_dict() if lock_handoff else None,
+ )
+ qm = get_queue_manager()
+ await qm.enqueue(QueueManager.EXTERNAL_PARSE, msg.to_dict())
+ await lock_lease.handoff()
+ monitor_started = True
+ except Exception as exc:
+ with contextlib.suppress(Exception):
+ await lock_lease.close()
+
+ if task is not None:
+ with contextlib.suppress(Exception):
+ await task_tracker.fail(
+ task.task_id,
+ str(exc),
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+
+ raise
+ logger.info(
+ "[ResourceService] Enqueued UnderstandingParseMsg task_id=%s root_uri=%s",
+ task.task_id,
+ root_uri,
+ )
+ await self._manage_watch_if_needed(
+ watch_manager=watch_manager,
+ skip_watch_management=skip_watch_management,
+ watch_interval=watch_interval,
+ target=target,
+ root_uri=root_uri,
+ path=path,
+ reason=reason,
+ instruction=instruction,
+ build_index=build_index,
+ summarize=summarize,
+ processor_kwargs=kwargs,
+ watch_auth_state=normalized_args.watch_auth_state,
+ ctx=ctx,
+ )
+ return {
+ "status": "success",
+ "root_uri": root_uri,
+ "task_id": task.task_id,
+ }
+
result = await self._resource_processor.process_resource(
path=path,
ctx=ctx,
@@ -690,54 +922,21 @@ class ResourceService:
root_uri=result.get("root_uri"),
)
telemetry.set("queue.wait.duration_ms", queue_wait_duration_ms)
- if watch_manager and not skip_watch_management:
- with telemetry.measure("resource.watch"):
- if watch_interval > 0:
- watch_to = target.to
- parent_uri = target.parent
- if not watch_to:
- watch_to = validate_optional_content_target_uri(
- result.get("root_uri"),
- ctx,
- kind="resource",
- field_name="root_uri",
- )
- parent_uri = None
- if not watch_to:
- raise InvalidArgumentError(
- "watch_interval > 0 requires a stable target URI. "
- "Pass 'to' explicitly, or add a resource type that returns root_uri."
- )
- try:
- processor_kwargs = self._sanitize_watch_processor_kwargs(kwargs)
- if normalized_args.watch_auth_state is not None:
- processor_kwargs.pop(FEISHU_ACCESS_TOKEN_ARG, None)
- await self._handle_watch_task_creation(
- path=path,
- to_uri=watch_to,
- parent_uri=parent_uri,
- reason=reason,
- instruction=instruction,
- watch_interval=watch_interval,
- build_index=build_index,
- summarize=summarize,
- processor_kwargs=processor_kwargs,
- auth_state=normalized_args.watch_auth_state,
- ctx=ctx,
- )
- except ConflictError:
- raise
- except Exception as e:
- logger.warning(
- f"[ResourceService] Failed to create watch task for {watch_to}: {e}"
- )
- elif target.to:
- try:
- await self._handle_watch_task_cancellation(to_uri=target.to, ctx=ctx)
- except Exception as e:
- logger.warning(
- f"[ResourceService] Failed to cancel watch task for {target.to}: {e}"
- )
+ await self._manage_watch_if_needed(
+ watch_manager=watch_manager,
+ skip_watch_management=skip_watch_management,
+ watch_interval=watch_interval,
+ target=target,
+ root_uri=str(result.get("root_uri") or ""),
+ path=path,
+ reason=reason,
+ instruction=instruction,
+ build_index=build_index,
+ summarize=summarize,
+ processor_kwargs=kwargs,
+ watch_auth_state=normalized_args.watch_auth_state,
+ ctx=ctx,
+ )
if wait:
await self._link_resource_reason_memory(
result=result,
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index 962e62291..3c40b5b58 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -30,6 +30,7 @@ def init_queue_manager(
mount_point: str = "/queue",
max_concurrent_embedding: int = 10,
max_concurrent_semantic: int = 64,
+ max_concurrent_external_parse: int = 4,
) -> "QueueManager":
"""Initialize QueueManager singleton.
@@ -47,6 +48,7 @@ def init_queue_manager(
mount_point=mount_point,
max_concurrent_embedding=max_concurrent_embedding,
max_concurrent_semantic=max_concurrent_semantic,
+ max_concurrent_external_parse=max_concurrent_external_parse,
)
return _instance
@@ -67,6 +69,7 @@ class QueueManager:
# Standard queue names
EMBEDDING = "Embedding"
SEMANTIC = "Semantic"
+ EXTERNAL_PARSE = "ExternalParse"
def __init__(
self,
@@ -75,6 +78,7 @@ class QueueManager:
mount_point: str = "/queue",
max_concurrent_embedding: int = 10,
max_concurrent_semantic: int = 64,
+ max_concurrent_external_parse: int = 4,
):
"""Initialize QueueManager."""
self._agfs = agfs
@@ -82,6 +86,7 @@ class QueueManager:
self.mount_point = mount_point
self._max_concurrent_embedding = max_concurrent_embedding
self._max_concurrent_semantic = max_concurrent_semantic
+ self._max_concurrent_external_parse = max_concurrent_external_parse
self._queues: Dict[str, NamedQueue] = {}
self._started = False
self._queue_threads: Dict[str, threading.Thread] = {}
@@ -149,11 +154,12 @@ class QueueManager:
if thread.is_alive():
return
- max_concurrent = (
- self._max_concurrent_embedding
- if queue.name == self.EMBEDDING
- else self._max_concurrent_semantic
- )
+ if queue.name == self.EMBEDDING:
+ max_concurrent = self._max_concurrent_embedding
+ elif queue.name == self.EXTERNAL_PARSE:
+ max_concurrent = self._max_concurrent_external_parse
+ else:
+ max_concurrent = self._max_concurrent_semantic
stop_event = threading.Event()
self._queue_stop_events[queue.name] = stop_event
thread = threading.Thread(
diff --git a/openviking/storage/queuefs/understanding_parse_msg.py b/openviking/storage/queuefs/understanding_parse_msg.py
new file mode 100644
index 000000000..12ea5c731
--- /dev/null
+++ b/openviking/storage/queuefs/understanding_parse_msg.py
@@ -0,0 +1,155 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""UnderstandingParseMsg: External parse queue message for UnderstandingAPI."""
+
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from typing import Any, Dict, Optional
+from uuid import uuid4
+
+
+@dataclass
+class UnderstandingParseMsg:
+ id: str
+ task_id: str
+ telemetry_id: Optional[str]
+ path: str
+ root_uri: str
+ lock_handoff: Optional[Dict[str, Any]] = None
+ status: str = "pending"
+ timestamp: int = int(datetime.now().timestamp())
+ account_id: str = "default"
+ user_id: str = "default"
+ role: str = "root"
+ actor_peer_id: Optional[str] = None
+ reason: str = ""
+ instruction: str = ""
+ build_index: bool = True
+ summarize: bool = False
+ strict: bool = False
+ ignore_dirs: Optional[str] = None
+ include: Optional[str] = None
+ exclude: Optional[str] = None
+ directly_upload_media: bool = True
+ allow_local_path_resolution: bool = True
+ enforce_public_remote_targets: bool = False
+ args: Dict[str, Any] = field(default_factory=dict)
+ source_name: Optional[str] = None
+
+ def __init__(
+ self,
+ *,
+ task_id: str,
+ path: str,
+ root_uri: str,
+ account_id: str,
+ user_id: str,
+ role: str,
+ actor_peer_id: Optional[str] = None,
+ telemetry_id: Optional[str] = None,
+ lock_handoff: Optional[Dict[str, Any]] = None,
+ reason: str = "",
+ instruction: str = "",
+ build_index: bool = True,
+ summarize: bool = False,
+ strict: bool = False,
+ ignore_dirs: Optional[str] = None,
+ include: Optional[str] = None,
+ exclude: Optional[str] = None,
+ directly_upload_media: bool = True,
+ allow_local_path_resolution: bool = True,
+ enforce_public_remote_targets: bool = False,
+ args: Optional[Dict[str, Any]] = None,
+ source_name: Optional[str] = None,
+ ):
+ self.id = str(uuid4())
+ self.task_id = task_id
+ self.telemetry_id = telemetry_id
+ self.path = path
+ self.root_uri = root_uri
+ self.account_id = account_id
+ self.user_id = user_id
+ self.role = role
+ self.actor_peer_id = actor_peer_id
+ self.lock_handoff = lock_handoff
+ self.reason = reason
+ self.instruction = instruction
+ self.build_index = build_index
+ self.summarize = summarize
+ self.strict = strict
+ self.ignore_dirs = ignore_dirs
+ self.include = include
+ self.exclude = exclude
+ self.directly_upload_media = directly_upload_media
+ self.allow_local_path_resolution = allow_local_path_resolution
+ self.enforce_public_remote_targets = enforce_public_remote_targets
+ self.args = args or {}
+ self.source_name = source_name
+
+ def to_dict(self) -> Dict[str, Any]:
+ return asdict(self)
+
+ def to_json(self) -> str:
+ return json.dumps(self.to_dict())
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> "UnderstandingParseMsg":
+ if not data:
+ raise ValueError("Data dictionary is empty")
+ task_id = data.get("task_id")
+ path = data.get("path")
+ root_uri = data.get("root_uri")
+ if not task_id or not path or not root_uri:
+ missing = []
+ if not task_id:
+ missing.append("task_id")
+ if not path:
+ missing.append("path")
+ if not root_uri:
+ missing.append("root_uri")
+ raise ValueError(f"Missing required fields: {missing}")
+
+ obj = cls(
+ task_id=str(task_id),
+ path=str(path),
+ root_uri=str(root_uri),
+ account_id=str(data.get("account_id", "default")),
+ user_id=str(data.get("user_id", "default")),
+ role=str(data.get("role", "root")),
+ actor_peer_id=data.get("actor_peer_id"),
+ telemetry_id=str(data.get("telemetry_id"))
+ if isinstance(data.get("telemetry_id"), str)
+ else None,
+ lock_handoff=data.get("lock_handoff")
+ if isinstance(data.get("lock_handoff"), dict)
+ else None,
+ reason=str(data.get("reason", "")),
+ instruction=str(data.get("instruction", "")),
+ build_index=bool(data.get("build_index", True)),
+ summarize=bool(data.get("summarize", False)),
+ strict=bool(data.get("strict", False)),
+ ignore_dirs=data.get("ignore_dirs"),
+ include=data.get("include"),
+ exclude=data.get("exclude"),
+ directly_upload_media=bool(data.get("directly_upload_media", True)),
+ allow_local_path_resolution=bool(data.get("allow_local_path_resolution", True)),
+ enforce_public_remote_targets=bool(data.get("enforce_public_remote_targets", False)),
+ args=data.get("args") if isinstance(data.get("args"), dict) else {},
+ source_name=data.get("source_name"),
+ )
+ if data.get("id"):
+ obj.id = str(data["id"])
+ if "status" in data:
+ obj.status = str(data["status"])
+ if "timestamp" in data:
+ obj.timestamp = int(data["timestamp"])
+ return obj
+
+ @classmethod
+ def from_json(cls, json_str: str) -> "UnderstandingParseMsg":
+ try:
+ data = json.loads(json_str)
+ except json.JSONDecodeError as e:
+ raise ValueError(f"Invalid JSON string: {e}")
+ return cls.from_dict(data)
diff --git a/openviking/storage/queuefs/understanding_parse_processor.py b/openviking/storage/queuefs/understanding_parse_processor.py
new file mode 100644
index 000000000..38f6e770b
--- /dev/null
+++ b/openviking/storage/queuefs/understanding_parse_processor.py
@@ -0,0 +1,344 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""UnderstandingParseProcessor: Processes UnderstandingParseMsg messages."""
+
+import asyncio
+import json
+
+from contextlib import suppress
+from typing import Any, Dict, Optional
+
+from openviking.observability.context import (
+ bind_execution_context,
+)
+from openviking.server.identity import RequestContext, Role
+from openviking.service.task_tracker import get_task_tracker
+from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.storage.queuefs.understanding_parse_msg import UnderstandingParseMsg
+from openviking.telemetry import bind_telemetry, bind_telemetry_stage, resolve_telemetry
+from openviking_cli.session.user_id import UserIdentifier
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class UnderstandingParseProcessor(DequeueHandlerBase):
+ def __init__(self, resource_processor: Any, resource_memory_link_service: Any = None):
+ self._resource_processor = resource_processor
+ self._resource_memory_link_service = resource_memory_link_service
+ self._background_tasks: set[asyncio.Task] = set()
+
+ async def _monitor_queue_then_link_memory(
+ self,
+ *,
+ task_id: str,
+ telemetry_id: str,
+ ctx: RequestContext,
+ result: Dict[str, Any],
+ reason: str,
+ source_name: Optional[str],
+ ) -> None:
+ from openviking.telemetry.resource_summary import unregister_wait_telemetry
+ from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
+
+ task_tracker = get_task_tracker()
+ request_wait_tracker = get_request_wait_tracker()
+ try:
+ request_wait_tracker.register_request(telemetry_id)
+ await request_wait_tracker.wait_for_request(telemetry_id)
+ status = request_wait_tracker.build_queue_status(telemetry_id)
+ errors = sum(int(group.get("error_count", 0) or 0) for group in status.values())
+ if errors:
+ await task_tracker.fail(
+ task_id,
+ f"queue processing failed: {status}",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error("queue processing failed", {"task_id": task_id})
+ return
+
+ result["queue_status"] = status
+ if self._resource_memory_link_service and (reason or "").strip():
+ root_uri = result.get("root_uri")
+ if root_uri:
+ try:
+ link_result = await self._resource_memory_link_service.on_resource_added(
+ ctx=ctx,
+ resource_uri=root_uri,
+ reason=reason,
+ source_name=source_name,
+ timeout=None,
+ )
+ result["memory_linking"] = link_result
+ except Exception as exc:
+ logger.warning(
+ "[UnderstandingParse] Failed to link resource reason memory: %s",
+ exc,
+ )
+ result.setdefault("warnings", []).append(f"Memory linking failed: {exc}")
+
+ await task_tracker.complete(
+ task_id,
+ result,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_success()
+ except Exception as exc:
+ await task_tracker.fail(
+ task_id,
+ str(exc),
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error(str(exc), {"task_id": task_id})
+ finally:
+ request_wait_tracker.cleanup(telemetry_id)
+ unregister_wait_telemetry(telemetry_id)
+
+ async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ if not data:
+ return None
+
+ payload: Dict[str, Any] = data
+ if isinstance(data, dict) and "data" in data:
+ inner = data.get("data")
+ if isinstance(inner, str) and inner.strip():
+ try:
+ decoded = json.loads(inner)
+ if isinstance(decoded, dict):
+ payload = decoded
+ except Exception:
+ payload = data
+ elif isinstance(inner, dict):
+ payload = inner
+ if isinstance(payload, dict) and data.get("id") and not payload.get("id"):
+ payload["id"] = data["id"]
+
+ msg = UnderstandingParseMsg.from_dict(payload)
+ ctx = RequestContext(
+ user=UserIdentifier(msg.account_id, msg.user_id),
+ role=Role(msg.role),
+ actor_peer_id=msg.actor_peer_id,
+ )
+
+ task_tracker = get_task_tracker()
+ lock_lease = None
+ if msg.lock_handoff is not None:
+ try:
+ from openviking.storage.transaction.lock_lease import (
+ LockHandoffRef,
+ OwnedLockLease,
+ )
+
+ ref = LockHandoffRef.from_value(msg.lock_handoff)
+ if ref is None:
+ raise ValueError("Invalid lock_handoff")
+ lock_lease = await OwnedLockLease.from_handoff(ref)
+ except Exception as exc:
+ retry_key = "_lock_handoff_retry"
+ raw_retry = (msg.args or {}).get(retry_key, 0)
+ try:
+ retry_count = int(raw_retry or 0)
+ except (TypeError, ValueError):
+ retry_count = 0
+
+ max_retries = 2
+ if retry_count < max_retries:
+ from openviking.storage.queuefs import QueueManager, get_queue_manager
+
+ qm = get_queue_manager()
+ if qm is None:
+ raise
+ new_args = dict(msg.args or {})
+ new_args[retry_key] = retry_count + 1
+ retry_msg = UnderstandingParseMsg(
+ task_id=msg.task_id,
+ telemetry_id=msg.telemetry_id,
+ path=msg.path,
+ root_uri=msg.root_uri,
+ account_id=msg.account_id,
+ user_id=msg.user_id,
+ role=msg.role,
+ actor_peer_id=msg.actor_peer_id,
+ lock_handoff=msg.lock_handoff,
+ reason=msg.reason,
+ instruction=msg.instruction,
+ build_index=msg.build_index,
+ summarize=msg.summarize,
+ strict=msg.strict,
+ ignore_dirs=msg.ignore_dirs,
+ include=msg.include,
+ exclude=msg.exclude,
+ directly_upload_media=msg.directly_upload_media,
+ allow_local_path_resolution=msg.allow_local_path_resolution,
+ enforce_public_remote_targets=msg.enforce_public_remote_targets,
+ args=new_args,
+ source_name=msg.source_name,
+ )
+ await qm.enqueue(QueueManager.EXTERNAL_PARSE, retry_msg.to_dict())
+ self.report_requeue()
+ self.report_success()
+ return None
+
+ await task_tracker.fail(
+ msg.task_id,
+ f"Invalid lock_handoff: {exc}",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error(f"Invalid lock_handoff: {exc}", data)
+ return None
+
+ logger.info(
+ "[UnderstandingParse] Dequeued task_id=%s root_uri=%s",
+ msg.task_id,
+ msg.root_uri,
+ )
+ resource_processor = self._resource_processor
+ if resource_processor is None:
+ raise RuntimeError("ResourceProcessor is None")
+
+ telemetry_id = msg.telemetry_id or ""
+ if telemetry_id:
+ from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
+
+ get_request_wait_tracker().register_request(telemetry_id)
+
+ telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None
+ if telemetry_id and telemetry is None:
+ from openviking.telemetry.operation import OperationTelemetry
+
+ telemetry = OperationTelemetry(operation="noop", enabled=False)
+ telemetry.telemetry_id = telemetry_id
+
+ with bind_execution_context(), (bind_telemetry(telemetry) if telemetry else suppress()):
+ try:
+ await task_tracker.start(
+ msg.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id
+ )
+ await task_tracker.update_stage(
+ msg.task_id,
+ "parsing",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+
+ bind_telemetry_stage("parse_understanding")
+ kwargs: Dict[str, Any] = {}
+ if lock_lease is not None and getattr(lock_lease, "active", False):
+ kwargs["resource_lock"] = lock_lease
+ if msg.enforce_public_remote_targets:
+ from openviking.utils.network_guard import ensure_public_remote_target
+
+ kwargs.setdefault("request_validator", ensure_public_remote_target)
+ result = await resource_processor.process_resource(
+ path=msg.path,
+ ctx=ctx,
+ reason=msg.reason,
+ instruction=msg.instruction,
+ scope="resources",
+ to=msg.root_uri,
+ parent=None,
+ build_index=msg.build_index,
+ summarize=msg.summarize,
+ allow_local_path_resolution=msg.allow_local_path_resolution,
+ enforce_public_remote_targets=msg.enforce_public_remote_targets,
+ strict=msg.strict,
+ source_name=msg.source_name,
+ ignore_dirs=msg.ignore_dirs,
+ include=msg.include,
+ exclude=msg.exclude,
+ directly_upload_media=msg.directly_upload_media,
+ args=msg.args,
+ skip_watch_management=True,
+ watch_interval=0,
+ **kwargs,
+ )
+
+ if result.get("status") == "error":
+ errors = result.get("errors") or ["resource processing failed"]
+ await task_tracker.fail(
+ msg.task_id,
+ "; ".join(str(error) for error in errors),
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error("resource processing failed", data)
+ return None
+
+ if telemetry_id:
+ await task_tracker.update_stage(
+ msg.task_id,
+ "processing_queue",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ monitor = asyncio.create_task(
+ self._monitor_queue_then_link_memory(
+ task_id=msg.task_id,
+ telemetry_id=telemetry_id,
+ ctx=ctx,
+ result=result,
+ reason=msg.reason,
+ source_name=msg.source_name,
+ )
+ )
+ self._background_tasks.add(monitor)
+ monitor.add_done_callback(self._background_tasks.discard)
+ return None
+
+ if self._resource_memory_link_service and (msg.reason or "").strip():
+ root_uri = result.get("root_uri")
+ if root_uri:
+ try:
+ link_result = await self._resource_memory_link_service.on_resource_added(
+ ctx=ctx,
+ resource_uri=root_uri,
+ reason=msg.reason,
+ source_name=msg.source_name,
+ timeout=None,
+ )
+ result["memory_linking"] = link_result
+ except Exception as exc:
+ logger.warning(
+ "[UnderstandingParse] Failed to link resource reason memory: %s",
+ exc,
+ )
+ result.setdefault("warnings", []).append(
+ f"Memory linking failed: {exc}"
+ )
+
+ await task_tracker.complete(
+ msg.task_id,
+ result,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_success()
+ return None
+ except asyncio.CancelledError:
+ await task_tracker.fail(
+ msg.task_id,
+ "external parse cancelled",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error("external parse cancelled", data)
+ raise
+ except Exception as exc:
+ await task_tracker.fail(
+ msg.task_id,
+ str(exc),
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error(str(exc), data)
+ return None
+ finally:
+ try:
+ if lock_lease is not None:
+ await lock_lease.close()
+ except Exception:
+ pass
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index f2ebf9e79..ab759659e 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -23,7 +23,8 @@ from openviking.storage.transaction import (
LockLease,
OwnedLockLease,
)
-from openviking.storage.viking_fs import get_viking_fs
+from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES
+from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.telemetry import get_current_telemetry
from openviking.utils.embedding_utils import index_resource
from openviking.utils.summarizer import Summarizer
@@ -298,6 +299,26 @@ class ResourceProcessor:
result["root_uri"] = root_uri
else:
target_preexisting = await viking_fs.exists(root_uri, ctx=ctx)
+ if target_preexisting:
+ try:
+ stat = await viking_fs.stat(root_uri, ctx=ctx)
+ if isinstance(stat, dict) and stat.get("isDir"):
+ entries = await viking_fs.ls(
+ root_uri,
+ show_all_hidden=True,
+ node_limit=LS_ALL_NODES,
+ ctx=ctx,
+ )
+ names: list[str] = []
+ for entry in entries:
+ name = entry.get("name", "")
+ if not name or name in {".", ".."}:
+ continue
+ names.append(str(name))
+ if all(name in STORAGE_INTERNAL_ENTRY_NAMES for name in names):
+ target_preexisting = False
+ except Exception:
+ pass
if not resource_lock.active:
dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)
resource_lock = await self.acquire_resource_lock(