Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

83.1 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Context Processor for OpenViking.

Handles coordinated writes and self-iteration processes
as described in the OpenViking design document.
"""

import asyncio
import inspect
import os
import time
from collections.abc import Awaitable, Callable
from dataclasses import replace
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union

from openviking.core.context import ContextLevel
from openviking.core.namespace import context_type_for_uri
from openviking.parse.image_rewrite import rewrite_image_uris
from openviking.parse.mode import ParseMode, normalize_parse_mode
from openviking.parse.tree_builder import TreeBuilder
from openviking.privacy.sanitizer import PrivacySanitizer
from openviking.resource.processing_mode import (
    DEFAULT_PROCESSING_MODE,
    VECTORS_ONLY,
    ProcessingMode,
    normalize_processing_mode,
)
from openviking.server.identity import RequestContext
from openviking.storage.acl import AclAction, AclSpec, AclUpdate
from openviking.storage.errors import LockAcquisitionError
from openviking.storage.expr import And, Eq, PathScope
from openviking.storage.index_action import FieldPatch
from openviking.storage.internal_names import is_storage_internal_name
from openviking.storage.pathlock_transfer import PathLockGuard
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
from openviking.storage.resource_rnfv import RequestIntent
from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.telemetry import bind_ingestion_source_type, get_current_telemetry
from openviking.utils import is_github_url
from openviking.utils.embedding_utils import (
    index_resource,
    vectorize_directory_meta,
    vectorize_file,
)
from openviking.utils.git_auth import is_git_https_url
from openviking.utils.ingest_options import IngestOptions
from openviking.utils.log_correlation import log_correlation
from openviking.utils.summarizer import Summarizer
from openviking_cli.exceptions import OpenVikingError
from openviking_cli.utils import VikingURI, get_logger
from openviking_cli.utils.config import get_openviking_config
from openviking_cli.utils.storage import StoragePath

if TYPE_CHECKING:
    from openviking.config.vlm import VLMResolver
    from openviking.parse.accessors.base import LocalResource
    from openviking.parse.vlm import VLMProcessor

logger = get_logger(__name__)

_PARSE_ARTIFACT_INTERNAL_NAMES = frozenset(
    {
        ".abstract.md",
        ".overview.md",
        ".relations.json",
        ".source.json",
        ".image_mappings.json",
        ".artifact_manifest.json",
    }
)
_MAX_FILE_VECTORIZATION_CONCURRENCY = 64
VECTORDB_MAX_QUERY_LIMIT = 100_000


class _DocRelStore:
    """Adapt a parse output store so commit reads use target-relative paths.

    Resource plans key files relative to the target root, while the underlying
    artifact may store them below ``<doc_rel>/...``.
    """

    def __init__(self, store: Any, doc_rel: str) -> None:
        self._store = store
        base = (doc_rel or "").strip("/")
        self._base = base

    async def read_bytes(self, ref: Any, rel_path: str) -> bytes:
        normalized = str(rel_path or "").strip("/")
        artifact_rel = (
            normalized
            if not self._base or normalized == self._base or normalized.startswith(self._base + "/")
            else f"{self._base}/{normalized}"
        )
        return await self._store.read_bytes(ref, artifact_rel)


class ResourceProcessor:
    """
    Handles coordinated write operations.

    When new data is added, automatically:
    1. Download if URL (prefer PDF format)
    2. Parse and structure the content (Parser writes to temp directory)
    3. Extract images/tables for mixed content
    4. Use VLM to understand non-text content
    5. TreeBuilder finalizes from temp (move to AGFS)
    6. SemanticQueue generates L0/L1 and vectorizes asynchronously
    """

    def __init__(
        self,
        vikingdb: VikingDBManager,
        media_storage: Optional["StoragePath"] = None,
        max_context_size: int = 2000,
        max_split_depth: int = 3,
        runtime_config_manager: Optional[Any] = None,
        vlm_resolver: Optional["VLMResolver"] = None,
    ):
        """Initialize coordinated writer."""
        self.vikingdb = vikingdb
        self.media_storage = media_storage
        self.runtime_config_manager = runtime_config_manager
        self.vlm_resolver = vlm_resolver
        self.tree_builder = TreeBuilder()
        self._vlm_processor = None
        self._media_processor = None
        self._summarizer = None
        self._privacy_sanitizer: PrivacySanitizer | None = None

    def set_privacy_sanitizer(self, sanitizer: PrivacySanitizer | None) -> None:
        self._privacy_sanitizer = sanitizer

    async def _sanitize_parse_artifact(
        self,
        *,
        store: Any,
        artifact_ref: Any,
        doc_rel: str,
        target_root_uri: str,
        ctx: RequestContext,
        root_is_file: bool = False,
    ) -> None:
        """Rewrite text in a temporary parse artifact before planning or persistence."""
        if self._privacy_sanitizer is None:
            return
        from openviking.parse.output import read_artifact_manifest, write_artifact_manifest
        from openviking.utils.content_hash import content_md5
        from openviking_cli.exceptions import InvalidArgumentError

        base = (doc_rel or "").strip("/")
        prefix = f"{base}/" if base else ""
        manifest = await read_artifact_manifest(store, artifact_ref)
        changed = False

        async def sanitize_file(artifact_path: str, relative: str) -> None:
            nonlocal changed
            data = await store.read_bytes(artifact_ref, artifact_path)
            try:
                content = data.decode("utf-8")
            except UnicodeDecodeError:
                return
            uri = (
                target_root_uri.rstrip("/")
                if not relative
                else f"{target_root_uri.rstrip('/')}/{relative}"
            )
            result = await self._privacy_sanitizer.sanitize(content, ctx=ctx, uri=uri)
            if result.ambiguous_references:
                raise InvalidArgumentError(
                    "Resource content contains a Privacy value mapped to multiple fields; "
                    "insert the intended owner-bound reference explicitly",
                    details={
                        "uri": uri,
                        "candidate_references": [
                            list(candidates) for candidates in result.ambiguous_references
                        ],
                    },
                )
            if result.content == content:
                return
            final_bytes = result.content.encode("utf-8")
            await store.write_bytes(artifact_ref, artifact_path, final_bytes)
            manifest[artifact_path] = content_md5(final_bytes)
            changed = True

        async def walk(directory: str) -> None:
            for entry in await store.list(artifact_ref, directory):
                if entry.is_dir:
                    await walk(entry.rel_path)
                    continue
                if entry.name in _PARSE_ARTIFACT_INTERNAL_NAMES or is_storage_internal_name(
                    entry.name
                ):
                    continue
                relative = (
                    entry.rel_path[len(prefix) :]
                    if prefix and entry.rel_path.startswith(prefix)
                    else entry.rel_path
                )
                await sanitize_file(entry.rel_path, relative)

        # A flattened document can be a file inside a directory artifact.
        if root_is_file or getattr(artifact_ref, "root_type", "dir") == "file":
            await sanitize_file(base, "")
        else:
            await walk(base)
        if changed:
            await write_artifact_manifest(store, artifact_ref, manifest)

    async def github_token_for(
        self,
        source: str,
        ctx: RequestContext,
    ) -> Optional[str]:
        """Resolve a GitHub token for one request without persisting it."""
        if not is_git_https_url(source) or not is_github_url(source):
            return None
        if self.runtime_config_manager is not None:
            account_github = await self.runtime_config_manager.get_account(ctx.account_id, "github")
            if account_github is not None and account_github.token:
                return account_github.token
        return os.environ.get("GITHUB_TOKEN") or None

    async def _source_config_kwargs(
        self,
        source: str,
        ctx: RequestContext,
        kwargs: dict[str, Any],
    ) -> dict[str, Any]:
        """Add account-scoped source configuration without persisting it."""
        from openviking.parse.accessors.feishu_accessor import FeishuAccessor

        if FeishuAccessor._is_feishu_url(source):
            return await self._feishu_source_config_kwargs(ctx, kwargs)
        return await self._github_source_config_kwargs(source, ctx, kwargs)

    async def _github_source_config_kwargs(
        self,
        source: str,
        ctx: RequestContext,
        kwargs: dict[str, Any],
    ) -> dict[str, Any]:
        """Add the account-scoped GitHub token for one request."""
        github_token = await self.github_token_for(source, ctx)
        if not github_token:
            return kwargs
        result = dict(kwargs)
        result["github_token"] = github_token
        return result

    async def _feishu_source_config_kwargs(
        self,
        ctx: RequestContext,
        kwargs: dict[str, Any],
    ) -> dict[str, Any]:
        """Add the account-scoped Feishu config for one request."""
        if self.runtime_config_manager is None:
            raise RuntimeError("Runtime config manager is not initialized")
        from openviking.config.feishu import get_effective_feishu_config

        feishu_config = await get_effective_feishu_config(
            self.runtime_config_manager,
            ctx.account_id,
        )
        result = dict(kwargs)
        result["feishu_config"] = feishu_config
        return result

    def _get_summarizer(self) -> "Summarizer":
        """Lazy initialization of the standalone Summarizer."""
        if self._summarizer is None:
            self._summarizer = Summarizer(self._get_vlm_processor())
        return self._summarizer

    async def _summarizer_for(self, ctx: RequestContext) -> "Summarizer":
        """Build a lightweight account-bound summarizer."""
        if self._summarizer is not None:
            return self._summarizer
        if self.vlm_resolver is None:
            raise RuntimeError("ResourceProcessor requires a VLM resolver for account-owned work")
        return Summarizer(await self._vlm_processor_for(ctx))

    def _get_vlm_processor(self) -> "VLMProcessor":
        """Return an explicitly configured standalone VLM processor."""
        if self._vlm_processor is None:
            raise RuntimeError("ResourceProcessor requires an explicitly configured VLMProcessor")
        return self._vlm_processor

    async def _vlm_processor_for(self, ctx: RequestContext) -> "VLMProcessor":
        from openviking.parse.vlm import VLMProcessor

        if self.vlm_resolver is None:
            raise RuntimeError("ResourceProcessor requires a VLM resolver for account-owned work")
        return VLMProcessor(vlm=await self.vlm_resolver.get_vlm(ctx.account_id))

    def _get_media_processor(self):
        """Lazy initialization of unified media processor."""
        if self._media_processor is None:
            from openviking.utils.media_processor import UnifiedResourceProcessor

            self._media_processor = UnifiedResourceProcessor(
                storage=self.media_storage,
            )
        return self._media_processor

    async def _media_processor_for(self, ctx: RequestContext):
        from openviking.utils.media_processor import UnifiedResourceProcessor

        if self.vlm_resolver is None and self._media_processor is not None:
            return self._media_processor
        if self.vlm_resolver is None:
            return self._get_media_processor()
        return UnifiedResourceProcessor(
            vlm_processor=await self._vlm_processor_for(ctx),
            storage=self.media_storage,
        )

    def _build_parse_output_store(self):
        """Return the configured parse output store, or None for AGFS mode.

        Local artifacts are consumed and committed to formal AGFS storage in
        this synchronous request phase. They never cross an asynchronous queue
        boundary, so parse and commit only need to run in the same process.
        AGFS (the default) returns None to preserve its temp-tree path.
        """
        try:
            parse_output = get_openviking_config().storage.parse_output
        except Exception:
            return None
        if getattr(parse_output, "mode", "agfs") != "local":
            return None
        from openviking.parse.output import build_parse_output_store

        return build_parse_output_store(
            backend="local", local_root=parse_output.resolved_local_root()
        )

    @staticmethod
    def _artifact_doc_rel(artifact_ref: Any, temp_doc_uri: Optional[str]) -> str:
        """Return the document root's path relative to the artifact root.

        ``temp_doc_uri`` is ``<artifact_root>/<doc_rel>`` (finalize built it), so
        the relative document path is the suffix after the artifact root.
        """
        root = artifact_ref.root.rstrip("/")
        doc = str(temp_doc_uri or "").rstrip("/")
        if doc.startswith(f"{root}/"):
            return doc[len(root) + 1 :]
        return str(getattr(artifact_ref, "resource_rel", "") or "").strip("/")

    @staticmethod
    def _ensure_parse_artifact_ref(parse_result: Any) -> Any:
        """Resolve an artifact ref for every supported parser result."""
        artifact_ref = getattr(parse_result, "artifact_ref", None)
        ensure = getattr(parse_result, "ensure_artifact_ref", None)
        if callable(ensure):
            artifact_ref = ensure()
        if artifact_ref is not None:
            return artifact_ref
        temp_dir_path = str(getattr(parse_result, "temp_dir_path", "") or "")
        if not temp_dir_path.startswith("viking://temp/"):
            raise RuntimeError("add_resources requires a parse artifact")
        from openviking.parse.output import ParseArtifactRef

        return ParseArtifactRef(backend="agfs", root=temp_dir_path, root_type="dir")

    @staticmethod
    async def _cleanup_parse_result_artifact(
        parse_result: Any, *, output_store: Any, viking_fs: Any, ctx: RequestContext
    ) -> None:
        artifact_ref = getattr(parse_result, "artifact_ref", None)
        if artifact_ref is not None:
            from openviking.parse.output import store_for_artifact_ref

            store = (
                output_store
                if output_store is not None and output_store.backend == artifact_ref.backend
                else store_for_artifact_ref(artifact_ref, viking_fs=viking_fs, ctx=ctx)
            )
            await store.cleanup(artifact_ref)
        elif parse_result.temp_dir_path:
            await viking_fs.delete_temp(parse_result.temp_dir_path, ctx=ctx)

    @staticmethod
    def _store_for_parse_artifact(
        artifact_ref: Any, *, output_store: Any, viking_fs: Any, ctx: RequestContext
    ) -> Any:
        if output_store is not None and output_store.backend == artifact_ref.backend:
            return output_store
        from openviking.parse.output import store_for_artifact_ref

        return store_for_artifact_ref(artifact_ref, viking_fs=viking_fs, ctx=ctx)

    async def _commit_directory_artifact_with_plan(
        self,
        *,
        output_store: Any,
        artifact_ref: Any,
        doc_rel: str,
        root_uri: str,
        target_preexisting: bool,
        root_is_file: bool = False,
        ctx: RequestContext,
        lease_ref: Optional[Dict[str, Any]],
        vectorize: bool,
        summarize: bool,
        processing_mode: ProcessingMode,
        is_code_repo: bool,
        ingest_options: IngestOptions,
        source_metadata: Optional[Dict[str, str]],
    ) -> Any:
        """Resolve and commit one artifact through the canonical update plan."""
        from openviking.metrics.datasources.resource import ResourceIngestionEventDataSource
        from openviking.storage.context_update_plan import (
            build_context_update_plan_from_snapshot,
            execute_content_tree_actions,
        )
        from openviking.storage.resource_diff import (
            build_rnfv_snapshot,
            count_tree_entry_kinds,
            prepare_artifact_inventory,
        )
        from openviking.storage.resource_target import AgfsResourceTarget

        target = AgfsResourceTarget(
            viking_fs=get_viking_fs(),
            root_uri=root_uri,
            ctx=ctx,
            lease_ref=lease_ref,
        )
        telemetry = get_current_telemetry()
        await self._sanitize_parse_artifact(
            store=output_store,
            artifact_ref=artifact_ref,
            doc_rel=doc_rel,
            target_root_uri=root_uri,
            ctx=ctx,
            root_is_file=root_is_file,
        )
        commit_store = _DocRelStore(output_store, doc_rel)
        artifact_backend = str(getattr(artifact_ref, "backend", "unknown"))
        update_plan_started_at = time.perf_counter()
        try:
            with telemetry.measure("resource.update_plan.artifact_inventory"):
                artifact_inventory = await prepare_artifact_inventory(
                    output_store,
                    artifact_ref,
                    doc_rel=doc_rel,
                    target_root_uri=root_uri,
                    root_is_file=root_is_file,
                )
            plan_processing_mode = (
                processing_mode
                if processing_mode == VECTORS_ONLY or summarize or vectorize
                else VECTORS_ONLY
            )
            request = RequestIntent.from_ingest_options(
                target_uri=root_uri,
                processing_mode=plan_processing_mode,
                ingest_options=ingest_options,
                vectorize=vectorize,
            )
            with telemetry.measure("resource.update_plan.rnfv_snapshot"):
                rnfv = await build_rnfv_snapshot(
                    viking_fs=get_viking_fs(),
                    vikingdb=self.vikingdb,
                    store=output_store,
                    artifact_ref=artifact_ref,
                    target_uri=root_uri,
                    ctx=ctx,
                    doc_rel=doc_rel,
                    request_intent=request,
                    target_preexisting=target_preexisting,
                    artifact_inventory=artifact_inventory,
                    root_is_file=root_is_file,
                )
            from collections import Counter

            n_files, n_dirs, _ = count_tree_entry_kinds(rnfv.new.entries)
            f_files, f_dirs, _ = count_tree_entry_kinds(rnfv.formal.entries)
            logger.debug(
                "[RNFVSnapshot] %s target=%s artifact_backend=%s root_is_file=%s "
                "target_preexisting=%s n_files=%d n_dirs=%d f_files=%d f_dirs=%d "
                "v_records=%d v_levels=%s",
                log_correlation(),
                root_uri,
                artifact_backend,
                root_is_file,
                target_preexisting,
                n_files,
                n_dirs,
                f_files,
                f_dirs,
                len(rnfv.vectors.records_by_id),
                dict(Counter(record.level for record in rnfv.vectors.records_by_id.values())),
            )
            with telemetry.measure("resource.update_plan.diff_and_compile"):
                diff, context_plan = await build_context_update_plan_from_snapshot(
                    snapshot=rnfv,
                    store=commit_store,
                    artifact_ref=artifact_ref,
                    target=target,
                    vikingdb=self.vikingdb,
                    context_type=context_type_for_uri(root_uri),
                    is_code_repo=is_code_repo,
                    account_id=ctx.account_id,
                    ctx=ctx,
                    root_preexisting=target_preexisting,
                    artifact_paths=artifact_inventory.artifact_paths,
                    ingest_options=ingest_options,
                    source_metadata=source_metadata,
                    root_is_file=root_is_file,
                )
        except Exception:
            telemetry.set(
                "resource.update_plan.duration_ms",
                (time.perf_counter() - update_plan_started_at) * 1000.0,
            )
            ResourceIngestionEventDataSource.record_stage(
                stage="update_plan",
                status="error",
                duration_seconds=time.perf_counter() - update_plan_started_at,
                account_id=ctx.account_id,
            )
            raise
        telemetry.set(
            "resource.update_plan.duration_ms",
            (time.perf_counter() - update_plan_started_at) * 1000.0,
        )
        ResourceIngestionEventDataSource.record_stage(
            stage="update_plan",
            status="ok",
            duration_seconds=time.perf_counter() - update_plan_started_at,
            account_id=ctx.account_id,
        )

        content_commit_started_at = time.perf_counter()
        try:
            with telemetry.measure("resource.content_commit"):
                await execute_content_tree_actions(
                    context_plan.content_tree_actions,
                    store=commit_store,
                    artifact_ref=artifact_ref,
                    target=target,
                )
        except Exception as exc:
            ResourceIngestionEventDataSource.record_stage(
                stage="content_commit",
                status="error",
                duration_seconds=time.perf_counter() - content_commit_started_at,
                account_id=ctx.account_id,
            )
            logger.exception(
                "[ContentTreeCommitFailed] %s target=%s artifact_backend=%s actions=%d error=%s",
                log_correlation(),
                root_uri,
                artifact_backend,
                len(context_plan.content_tree_actions),
                exc,
            )
            raise
        ResourceIngestionEventDataSource.record_stage(
            stage="content_commit",
            status="ok",
            duration_seconds=time.perf_counter() - content_commit_started_at,
            account_id=ctx.account_id,
        )
        self._log_content_tree_commit(
            root_uri=root_uri,
            artifact_backend=artifact_backend,
            actions=context_plan.content_tree_actions,
            duration_ms=(time.perf_counter() - content_commit_started_at) * 1000.0,
        )
        self._log_context_update_plan(context_plan)
        self._log_context_commit_summary(
            diff,
            content_actions=context_plan.content_tree_actions,
            root_uri=root_uri,
            is_initial=not target_preexisting,
        )
        return context_plan.after_content_commit()

    @staticmethod
    def _log_content_tree_commit(
        *, root_uri: str, artifact_backend: str, actions: Any, duration_ms: float
    ) -> None:
        log = logger.info if actions else logger.debug
        log(
            "[ContentTreeCommit] %s target=%s artifact_backend=%s uploaded_files=%d "
            "created_dirs=%d deleted_paths=%d replaced_kinds=%d duration_ms=%.3f",
            log_correlation(),
            root_uri,
            artifact_backend,
            sum(action.new_kind == "file" for action in actions),
            sum(action.new_kind == "directory" for action in actions),
            sum(action.operation.value == "delete" for action in actions),
            sum(action.operation.value == "replace_kind" for action in actions),
            duration_ms,
        )

    @staticmethod
    def _log_context_update_plan(plan: Any) -> None:
        from collections import Counter

        content = Counter(action.operation.value for action in plan.content_tree_actions)
        direct = Counter(action.action.value for action in plan.direct_index_actions)
        entries = plan.semantic_plan.tree.entries if plan.semantic_plan is not None else ()
        semantic = Counter(entry.semantic_action.value for entry in entries)
        slots = Counter(slot.action.value for entry in entries for slot in entry.index_slots)
        logger.debug(
            "[ContextUpdatePlan] %s root=%s content=%s semantic=%s direct_index=%s "
            "index_slots=%s "
            "semantic_entries=%d execution_roots=%d",
            log_correlation(),
            plan.root_uri,
            dict(content),
            dict(semantic),
            dict(direct),
            dict(slots),
            len(entries),
            len(plan.semantic_plan.execution_root_uris()) if plan.semantic_plan is not None else 0,
        )

    @staticmethod
    def _log_context_commit_summary(
        diff: Any, *, content_actions: Any, root_uri: str, is_initial: bool
    ) -> None:
        from collections import Counter

        states = Counter(entry.content_state.value for entry in diff.entries.values())
        uploaded_files = sum(action.new_kind == "file" for action in content_actions)
        created_dirs = sum(action.new_kind == "directory" for action in content_actions)
        if is_initial:
            logger.debug(
                "[add_resource] %s initial import committed root=%s files=%d dirs=%d",
                log_correlation(),
                root_uri,
                uploaded_files,
                created_dirs,
            )
            return
        logger.debug(
            "[add_resource] %s incremental diff committed root=%s states=%s uploaded=%d",
            log_correlation(),
            root_uri,
            dict(states),
            uploaded_files,
        )

    @staticmethod
    def _empty_directory_error(meta: Dict[str, Any]) -> str:
        """Build a bounded error message for a directory with no successful files."""
        failed_files = meta.get("failed_files")
        failures = failed_files if isinstance(failed_files, list) else []
        try:
            total_processable = int(meta.get("total_processable", 0) or 0)
        except (TypeError, ValueError):
            total_processable = 0

        if total_processable > 0:
            message = (
                "Directory import produced no content: "
                f"all {total_processable} processable file(s) failed"
            )
        else:
            message = "Directory import produced no content: no processable files were selected"

        details = ResourceProcessor._failed_file_details(failures)
        if details:
            message += "; failed files: " + "; ".join(details)
            if len(failures) > len(details):
                message += f"; ... {len(failures) - len(details)} more"
        return message

    @staticmethod
    def _directory_parse_failures(meta: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Return files selected for parsing whose parser did not produce content."""
        failed_files = meta.get("failed_files")
        if not isinstance(failed_files, list):
            return []
        return [
            item
            for item in failed_files
            if isinstance(item, dict)
            and (item.get("status") == "failed" or (not item.get("status") and "error" in item))
        ]

    @staticmethod
    def _incomplete_directory_error(failures: List[Dict[str, Any]]) -> str:
        message = f"Directory import incomplete: {len(failures)} file(s) failed to parse"
        details = ResourceProcessor._failed_file_details(failures)
        if details:
            message += "; failed files: " + "; ".join(details)
            if len(failures) > len(details):
                message += f"; ... {len(failures) - len(details)} more"
        return message

    @staticmethod
    def _failed_file_details(failures: List[Dict[str, Any]]) -> List[str]:
        details: List[str] = []
        for item in failures[:5]:
            if not isinstance(item, dict):
                continue
            path = str(item.get("path") or "<unknown>")
            reason = str(item.get("error") or item.get("reason") or "failed")
            remote_ids = [
                f"{key}={item[key]}" for key in ("file_id", "response_id") if item.get(key)
            ]
            if remote_ids:
                path = f"{path} ({', '.join(remote_ids)})"
            details.append(f"{path}: {reason[:120]}")
        return details

    async def prepare_durable_source(
        self,
        path: str,
        ctx: RequestContext,
        *,
        snapshot_required: bool = False,
        allow_local_path_resolution: bool = True,
        **kwargs,
    ) -> Optional["LocalResource"]:
        """Freeze a source when durable routing cannot safely defer access."""
        media_processor = await self._media_processor_for(ctx)
        if not snapshot_required and not media_processor.durable_route_requires_preparation(
            path, **kwargs
        ):
            return None
        kwargs = await self._source_config_kwargs(path, ctx, kwargs)
        with get_viking_fs().bind_request_context(ctx):
            return await media_processor.prepare(
                path,
                allow_local_path_resolution=allow_local_path_resolution,
                **kwargs,
            )

    def understanding_api_enabled(self) -> bool:
        return self._get_media_processor().understanding_api_enabled()

    def should_use_understanding_api(self, source: Union[str, "LocalResource"]) -> bool:
        return self._get_media_processor().should_use_understanding_api(source)

    def should_use_understanding_directly(self, source: str, **kwargs) -> bool:
        return self._get_media_processor().should_use_understanding_directly(source, **kwargs)

    async def submit_understanding(self, source: Union[str, "LocalResource"], **kwargs) -> str:
        return await self._get_media_processor().submit_understanding(source, **kwargs)

    async def upload_understanding_file(self, source: Union[str, "LocalResource"]) -> str:
        return await self._get_media_processor().upload_understanding_file(source)

    async def build_index(
        self, resource_uris: List[str], ctx: RequestContext, **kwargs
    ) -> Dict[str, Any]:
        """Expose index building as a standalone method."""
        ingest_options = IngestOptions.from_value(kwargs.get("ingest_options"))
        if ingest_options.search_tags is None and kwargs.get("search_tags") is not None:
            ingest_options = IngestOptions.from_search_tags(
                kwargs.get("search_tags"),
                mode=kwargs.get("search_tag_mode", "replace"),
            )
        for uri in resource_uris:
            await index_resource(
                uri,
                ctx,
                ingest_options=ingest_options,
            )
        return {"status": "success", "message": f"Indexed {len(resource_uris)} resources"}

    async def summarize(
        self, resource_uris: List[str], ctx: RequestContext, **kwargs
    ) -> Dict[str, Any]:
        """Expose summarization as a standalone method."""
        return await (await self._summarizer_for(ctx)).summarize(resource_uris, ctx, **kwargs)

    async def process_resource(
        self,
        path: str,
        ctx: RequestContext,
        reason: str = "",
        instruction: str = "",
        scope: str = "resources",
        user: Optional[str] = None,
        to: Optional[str] = None,
        parent: Optional[str] = None,
        summarize: bool = False,
        stage_callback: Optional[Callable[[str], Any]] = None,
        prepared_resource: Optional["LocalResource"] = None,
        tags: Optional[List[str]] = None,
        tag_mode: str = "replace",
        acl: AclSpec | Dict[str, Any] | None = None,
        target_validator: Optional[Callable[[str, RequestContext], Awaitable[None]]] = None,
        **kwargs,
    ) -> Dict[str, Any]:
        """
        Process and store a new resource.

        Workflow:
        1. Parse source (writes to temp directory)
        2. TreeBuilder builds final URI metadata
        3. Source commit moves temp content to the final path
        4. (Optional) Build vector index
        5. (Optional) Summarize

        target_validator rejects the resolved destination before reservation or
        source commit; its errors propagate after temporary artifact cleanup.
        """
        result = {
            "status": "success",
            "errors": [],
            "source_path": None,
        }
        defer_post_processing = bool(kwargs.pop("defer_post_processing", False))
        preacquired_lock = kwargs.pop("resource_lock", None)
        ingest_options = IngestOptions.from_search_tags(tags, mode=tag_mode)
        to_is_directory = bool(kwargs.pop("to_is_directory", False))
        telemetry = get_current_telemetry()
        metrics_account_id = getattr(ctx, "account_id", None)

        async def _set_stage(stage: str) -> None:
            if stage_callback is None:
                return
            result = stage_callback(stage)
            if inspect.isawaitable(result):
                await result

        with telemetry.measure("resource.source_execute"):
            # ============ Phase 1: Parse source and writes to temp viking fs ============
            try:
                from openviking.metrics.datasources.resource import (
                    ResourceIngestionEventDataSource,
                )

                media_processor = await self._media_processor_for(ctx)
                viking_fs = get_viking_fs()
                # Use reason as instruction fallback so it influences L0/L1
                # generation and improves search relevance as documented.
                effective_instruction = instruction or reason
                # Local artifact mode writes parse output to this worker's disk
                # instead of AGFS temp. The same synchronous request commits all
                # required bytes to formal AGFS before any async queue handoff.
                # The store is task-scoped; default AGFS mode leaves kwargs untouched.
                output_store = self._build_parse_output_store()
                if output_store is not None:
                    kwargs.setdefault("parse_output_store", output_store)
                if path.startswith(("http://", "https://", "git@", "ssh://", "git://")):
                    await _set_stage("fetching")
                else:
                    await _set_stage("parsing")
                kwargs = await self._source_config_kwargs(path, ctx, kwargs)
                with viking_fs.bind_request_context(ctx):
                    parse_result = await media_processor.process(
                        source=path,
                        instruction=effective_instruction,
                        prepared_resource=prepared_resource,
                        _metrics_account_id=metrics_account_id,
                        **kwargs,
                    )
                result["source_path"] = parse_result.source_path or path
                result["meta"] = parse_result.meta

                # Only abort when no temp content was produced at all.
                # For directory imports partial success (some files failed) is
                # normal - finalization should still proceed.
                if not parse_result.temp_dir_path:
                    result["status"] = "error"
                    result["errors"].extend(
                        parse_result.warnings or ["Parse failed: no content generated"],
                    )
                    return result

                parse_meta = parse_result.meta if isinstance(parse_result.meta, dict) else {}
                is_directory_aggregate = all(
                    key in parse_meta
                    for key in (
                        "file_count",
                        "total_processable",
                        "processed_files",
                        "failed_files",
                    )
                )
                if is_directory_aggregate and parse_meta.get("file_count") == 0:
                    result["status"] = "error"
                    result["errors"].append(self._empty_directory_error(parse_meta))
                    try:
                        await self._cleanup_parse_result_artifact(
                            parse_result, output_store=output_store, viking_fs=viking_fs, ctx=ctx
                        )
                    except Exception as exc:
                        logger.warning(
                            "[ResourceProcessor] Failed to clean empty directory temp %s: %s",
                            parse_result.temp_dir_path,
                            exc,
                        )
                    return result

                parse_failures = self._directory_parse_failures(parse_meta)
                if is_directory_aggregate and parse_failures:
                    result["status"] = "error"
                    result["errors"].append(self._incomplete_directory_error(parse_failures))
                    try:
                        await self._cleanup_parse_result_artifact(
                            parse_result, output_store=output_store, viking_fs=viking_fs, ctx=ctx
                        )
                    except Exception as exc:
                        logger.warning(
                            "[ResourceProcessor] Failed to clean incomplete directory temp %s: %s",
                            parse_result.temp_dir_path,
                            exc,
                        )
                    return result

                if parse_result.warnings and kwargs.get("strict", False):
                    result.setdefault("warnings", []).extend(parse_result.warnings)

            except OpenVikingError:
                raise
            except Exception as e:
                result["status"] = "error"
                error_message = f"Parse error: {e}"
                error_meta = getattr(e, "meta", {})
                if isinstance(error_meta, dict) and error_meta.get("response_id"):
                    error_message += f" (response_id={error_meta['response_id']})"
                result["errors"].append(error_message)
                logger.error(f"[ResourceProcessor] Parse error: {e}")
                telemetry.set_error("resource_processor.parse_artifact", "PROCESSING_ERROR", str(e))
                import traceback

                traceback.print_exc()
                return result

            # parse_result contains:
            # - root: ResourceNode tree (with L0/L1 in meta)
            # - temp_dir_path: Temporary directory path (Parser wrote all files)
            # - source_path, source_format

            # ============ Phase 3: TreeBuilder finalizes from temp (scan + move to AGFS) ============
            try:
                await _set_stage("target_resolve")
                stage_start = time.perf_counter()
                stage_status = "ok"
                finalize_start = time.perf_counter()
                artifact_ref = getattr(parse_result, "artifact_ref", None)
                artifact_output_store = (
                    self._store_for_parse_artifact(
                        artifact_ref,
                        output_store=output_store,
                        viking_fs=viking_fs,
                        ctx=ctx,
                    )
                    if artifact_ref is not None
                    else None
                )
                with get_viking_fs().bind_request_context(ctx):
                    context_tree = await self.tree_builder.finalize_from_temp(
                        temp_dir_path=parse_result.temp_dir_path,
                        ctx=ctx,
                        scope=scope,
                        to_uri=to,
                        parent_uri=parent,
                        source_path=parse_result.source_path,
                        source_format=parse_result.source_format,
                        create_parent=kwargs.get("create_parent", False),
                        flatten_single_file=(
                            normalize_parse_mode(kwargs.get("parse_mode", ParseMode.DEFAULT))
                            is ParseMode.NO_SPLIT
                            and parse_result.source_format not in {"directory", "repository"}
                            and not to_is_directory
                        ),
                        artifact_ref=artifact_ref,
                        output_store=artifact_output_store,
                    )
                    if context_tree and context_tree.root:
                        result["root_uri"] = context_tree.root.uri
                        result["temp_uri"] = context_tree.root.temp_uri
                    root_is_file = bool(getattr(context_tree, "_root_is_file", False))
                telemetry.set(
                    "resource.target_resolve.duration_ms",
                    round((time.perf_counter() - finalize_start) * 1000, 3),
                )
            except Exception as e:
                result["status"] = "error"
                result["errors"].append(f"Finalize from temp error: {e}")
                telemetry.set_error("resource_processor.target_resolve", "PROCESSING_ERROR", str(e))
                stage_status = "error"

                # Cleanup the parser-owned artifact through its own backend.
                try:
                    await self._cleanup_parse_result_artifact(
                        parse_result,
                        output_store=output_store,
                        viking_fs=get_viking_fs(),
                        ctx=ctx,
                    )
                except Exception:
                    pass

                return result
            finally:
                try:
                    ResourceIngestionEventDataSource.record_stage(
                        stage="target_resolve",
                        status=str(stage_status),
                        duration_seconds=float(time.perf_counter() - stage_start),
                        account_id=getattr(ctx, "account_id", None),
                    )
                except Exception:
                    pass

            # ============ Phase 3.5: Source commit + resource lock ============
            root_uri = result.get("root_uri")
            temp_uri = result.get("temp_uri")  # temp_doc_uri
            original_temp_uri = temp_uri  # 保存原始 temp_uri 用于最终输出
            candidate_uri = getattr(context_tree, "_candidate_uri", None) if context_tree else None
            resource_lock: Optional[Dict[str, Any]] = preacquired_lock
            target_preexisting = False
            source_committed = False
            local_artifact_doc_rel = ""
            incremental_noop = False
            context_update_plan = None

            if root_uri and temp_uri:
                viking_fs = get_viking_fs()
                try:
                    if candidate_uri:
                        if resource_lock is not None:
                            root_uri = candidate_uri
                            if target_validator is not None:
                                await target_validator(root_uri, ctx)
                        else:
                            root_uri, resource_lock = await self.reserve_unique_candidate(
                                candidate_uri=candidate_uri,
                                ctx=ctx,
                                root_is_file=root_is_file,
                                target_validator=target_validator,
                            )
                            result["root_uri"] = root_uri
                            if root_uri != candidate_uri:
                                result.setdefault("warnings", []).append(
                                    f"'{candidate_uri}' already exists. Creating '{root_uri}'. "
                                    f"Tip: Use --to <path> to specify exact target."
                                )
                    else:
                        if target_validator is not None:
                            await target_validator(root_uri, ctx)
                        target_preexisting = await viking_fs.exists(root_uri, ctx=ctx)
                        if target_preexisting:
                            try:
                                stat = await viking_fs.stat(root_uri, ctx=ctx, skip_count=True)
                                if isinstance(stat, dict) and stat.get("isDir"):
                                    entries = await viking_fs.ls(
                                        root_uri,
                                        show_all_hidden=True,
                                        node_limit=LS_ALL_NODES,
                                        ctx=ctx,
                                    )
                                    names: list[str] = []
                                    for entry in entries:
                                        name = entry.get("name", "")
                                        if not name or name in {".", ".."}:
                                            continue
                                        names.append(str(name))
                                    if all(is_storage_internal_name(name) for name in names):
                                        target_preexisting = False
                            except Exception:
                                pass
                        if resource_lock is None:
                            dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)
                            resource_lock = await self.acquire_resource_lock(
                                dst_path,
                                uri=root_uri,
                                root_is_file=root_is_file,
                            )
                    if acl is not None:
                        ingest_options = replace(
                            ingest_options,
                            acl_update=await viking_fs.prepare_acl_update(root_uri, acl, ctx),
                        )
                    artifact_ref = self._ensure_parse_artifact_ref(parse_result)
                    artifact_store = self._store_for_parse_artifact(
                        artifact_ref, output_store=output_store, viking_fs=viking_fs, ctx=ctx
                    )
                    local_artifact_doc_rel = self._artifact_doc_rel(artifact_ref, temp_uri)
                    context_update_plan = await self._commit_directory_artifact_with_plan(
                        output_store=artifact_store,
                        artifact_ref=artifact_ref,
                        doc_rel=local_artifact_doc_rel,
                        root_uri=root_uri,
                        target_preexisting=target_preexisting,
                        root_is_file=root_is_file,
                        ctx=ctx,
                        lease_ref=resource_lock,
                        vectorize=bool(kwargs.get("build_index", True)),
                        summarize=summarize,
                        processing_mode=normalize_processing_mode(kwargs.get("processing_mode")),
                        ingest_options=ingest_options,
                        is_code_repo=parse_result.source_format == "repository",
                        source_metadata=self._semantic_source_metadata(
                            path=path,
                            prepared_resource=prepared_resource,
                            source_format=parse_result.source_format,
                        ),
                    )
                    incremental_noop = target_preexisting and context_update_plan.is_noop()
                    temp_uri = root_uri
                    source_committed = True
                except BaseException:
                    # Mirror the Phase 3 (finalize) on-error cleanup: a lock or
                    # persist failure here would otherwise orphan the
                    # viking://temp tree with no GC (#2478). Skip when the temp
                    # tree was already persisted + deleted on the success path.
                    try:
                        if not source_committed:
                            await self._cleanup_parse_result_artifact(
                                parse_result,
                                output_store=output_store,
                                viking_fs=get_viking_fs(),
                                ctx=ctx,
                            )
                    except Exception:
                        pass
                    finally:
                        # The caller still owns a preacquired lease. Release
                        # leases acquired here if commit fails before handoff.
                        if resource_lock is not None and preacquired_lock is None:
                            await viking_fs._async_agfs.pathlock_release(resource_lock)
                    raise

            if artifact_ref is not None:
                artifact_store = self._store_for_parse_artifact(
                    artifact_ref,
                    output_store=output_store,
                    viking_fs=get_viking_fs(),
                    ctx=ctx,
                )
                try:
                    await artifact_store.cleanup(artifact_ref)
                except Exception as exc:
                    logger.warning(
                        "[ResourceProcessor] Failed to clean committed parse artifact %s: %s",
                        artifact_ref.root,
                        exc,
                    )
                artifact_ref = None
            prepared_artifact_ref = artifact_ref.to_dict() if artifact_ref is not None else None
            if prepared_artifact_ref is not None and artifact_ref.backend == "local":
                prepared_artifact_ref["resource_rel"] = local_artifact_doc_rel
            prepared = {
                "root_uri": root_uri,
                "temp_uri": temp_uri or parse_result.temp_dir_path,
                "temp_dir_path": parse_result.temp_dir_path,
                "artifact_ref": prepared_artifact_ref,
                "source_committed": source_committed,
                "target_preexisting": target_preexisting,
                "is_code_repo": parse_result.source_format == "repository",
                "root_is_file": root_is_file,
                "incremental_noop": incremental_noop,
                "acl_update": ingest_options.acl_update.model_dump(mode="json")
                if ingest_options.acl_update
                else None,
                "context_update_plan": (
                    context_update_plan.to_dict() if context_update_plan is not None else None
                ),
                "semantic_source": self._semantic_source_metadata(
                    path=path,
                    prepared_resource=prepared_resource,
                    source_format=parse_result.source_format,
                ),
            }
            if defer_post_processing:
                result["_post_process"] = prepared
                result["_resource_lock"] = resource_lock
            else:
                post_result = await self.finish_prepared_resource(
                    prepared,
                    ctx=ctx,
                    resource_lock=resource_lock,
                    summarize=summarize,
                    ingest_options=ingest_options,
                    build_index=bool(kwargs.get("build_index", True)),
                    processing_mode=normalize_processing_mode(kwargs.get("processing_mode")),
                )
                if post_result.get("warnings"):
                    result.setdefault("warnings", []).extend(post_result["warnings"])

            # 恢复原始 temp_uri 用于输出
            if original_temp_uri is not None:
                result["temp_uri"] = original_temp_uri

            return result

    async def finish_prepared_resource(
        self,
        prepared: Dict[str, Any],
        *,
        ctx: RequestContext,
        resource_lock: Optional[Dict[str, Any]] = None,
        summarize: bool = False,
        processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,
        build_index: bool = True,
        ingest_options: IngestOptions | None = None,
    ) -> Dict[str, Any]:
        """Run the queue-producing phase for a resource already stored in VikingFS."""
        from openviking.metrics.datasources.resource import ResourceIngestionEventDataSource

        root_uri = str(prepared.get("root_uri") or "")
        temp_uri = prepared.get("temp_uri")
        temp_dir_path = prepared.get("temp_dir_path")
        # Canonical plans contain only final resource URIs. An artifact ref is
        # present here only for legacy non-plan paths that still need it.
        artifact_ref_data = prepared.get("artifact_ref")
        artifact_ref = None
        if artifact_ref_data is not None:
            from openviking.parse.output import ParseArtifactRef

            artifact_ref = ParseArtifactRef.from_dict(artifact_ref_data)
            if artifact_ref.backend not in {"agfs", "local"}:
                raise ValueError(
                    f"Unsupported parse artifact backend in post-process: {artifact_ref.backend}"
                )
        source_committed = bool(prepared.get("source_committed"))
        metrics_account_id = getattr(ctx, "account_id", None)
        target_preexisting = bool(prepared.get("target_preexisting"))
        processing_mode = normalize_processing_mode(processing_mode)
        vectors_only = processing_mode == VECTORS_ONLY
        root_is_file = bool(prepared.get("root_is_file"))
        source_type = "repository" if bool(prepared.get("is_code_repo")) else "document"
        ingest_options = ingest_options or IngestOptions()
        if prepared.get("acl_update") is not None:
            ingest_options = replace(
                ingest_options, acl_update=AclUpdate.model_validate(prepared["acl_update"])
            )
        semantic_source = prepared.get("semantic_source")
        context_update_plan_data = prepared.get("context_update_plan")
        context_update_plan = None
        if context_update_plan_data is not None:
            from openviking.storage.context_update_plan import ContextUpdatePlan

            context_update_plan = ContextUpdatePlan.from_dict(context_update_plan_data)
            semantic_plan = (
                context_update_plan.semantic_plan.to_dict()
                if context_update_plan.semantic_plan is not None
                else None
            )
            direct_index_actions = context_update_plan.direct_index_actions
        else:
            semantic_plan = None
            direct_index_actions = ()
        uses_context_update_plan = context_update_plan_data is not None
        should_summarize = (
            not root_is_file
            and not vectors_only
            and (summarize or build_index)
            and (not uses_context_update_plan or semantic_plan is not None)
        )
        if context_update_plan is not None:
            file_refresh = context_update_plan.file_refresh
        elif root_is_file and source_committed and not vectors_only and (summarize or build_index):
            from openviking.storage.context_update_plan import FileRefreshIntent

            file_refresh = FileRefreshIntent(
                root_uri, (prepared.get("file_md5s") or {}).get(root_uri)
            )
        else:
            file_refresh = None
        should_refresh_file_parent = file_refresh is not None and not vectors_only
        result: Dict[str, Any] = {"status": "success", "root_uri": root_uri}
        artifact_cleaned = False

        async def cleanup_artifact_if_owned() -> None:
            nonlocal artifact_cleaned
            if artifact_cleaned or artifact_ref is None:
                return
            from openviking.parse.output import AgfsParseOutputStore

            # Local mode reuses the processor's own store seam (test-injectable);
            # agfs mode builds a temp-backed store on demand.
            output_store = (
                self._build_parse_output_store()
                if artifact_ref.backend == "local"
                else AgfsParseOutputStore(viking_fs=get_viking_fs(), ctx=ctx)
            )
            if output_store is not None:
                await output_store.cleanup(artifact_ref)
                artifact_cleaned = True

        derived_enqueue_started_at = time.perf_counter()
        derived_enqueue_status = "ok"
        try:
            with get_current_telemetry().measure("resource.derived_enqueue"):
                if prepared.get("incremental_noop") and not direct_index_actions:
                    if ingest_options.acl_update:
                        await get_viking_fs().acl_manager.apply_indexed_update(
                            ingest_options.acl_update, ctx
                        )
                    await cleanup_artifact_if_owned()
                    if resource_lock is not None:
                        await get_viking_fs()._async_agfs.pathlock_release(resource_lock)
                        resource_lock = None
                    return result

                if direct_index_actions:
                    with bind_ingestion_source_type(source_type):
                        await self._enqueue_index_actions(
                            direct_index_actions, ctx=ctx, ingest_options=ingest_options
                        )

                if should_summarize:
                    try:
                        summary_guard = (
                            PathLockGuard(get_viking_fs()._async_agfs, resource_lock)
                            if resource_lock is not None
                            else None
                        )
                        try:
                            summary_result = await (await self._summarizer_for(ctx)).summarize(
                                resource_uris=[root_uri],
                                ctx=ctx,
                                skip_vectorization=not build_index,
                                lock=summary_guard,
                                temp_uris=[temp_uri],
                                is_code_repo=bool(prepared.get("is_code_repo")),
                                ingest_options=ingest_options,
                                semantic_source=semantic_source,
                                generation_trigger="resource_ingest",
                                semantic_plan=semantic_plan,
                            )
                        finally:
                            if summary_guard is not None:
                                resource_lock = summary_guard.lease
                        if semantic_plan is not None and summary_result.get("status") != "success":
                            raise RuntimeError(
                                str(summary_result.get("message") or "semantic plan enqueue failed")
                            )
                        if (
                            resource_lock is not None
                            and summary_result.get("status") == "success"
                            and summary_result.get("enqueued_count", 0) > 0
                        ):
                            if (
                                not summary_result.get("lock_handed_off", False)
                                and summary_guard is not None
                            ):
                                try:
                                    await summary_guard.handoff()
                                finally:
                                    resource_lock = summary_guard.lease
                            resource_lock = None
                        if semantic_plan is not None and (
                            summary_result.get("status") == "success"
                            and summary_result.get("enqueued_count", 0) > 0
                        ):
                            await cleanup_artifact_if_owned()
                    except Exception as exc:
                        logger.error("Semantic enqueue failed: %s", exc)
                        if semantic_plan is not None:
                            raise
                        result["warnings"] = [f"Semantic enqueue failed: {exc}"]
        except BaseException:
            derived_enqueue_status = "error"
            try:
                await cleanup_artifact_if_owned()
            finally:
                if resource_lock is not None:
                    await get_viking_fs()._async_agfs.pathlock_release(resource_lock)
                    resource_lock = None
            raise
        finally:
            ResourceIngestionEventDataSource.record_stage(
                stage="derived_enqueue",
                status=derived_enqueue_status,
                duration_seconds=time.perf_counter() - derived_enqueue_started_at,
                account_id=metrics_account_id,
            )

        if resource_lock is not None:
            try:
                sync_deleted_files: list[str] = []
                sync_deleted_dirs: list[str] = []
                if not should_summarize and temp_uri and not source_committed:
                    viking_fs = get_viking_fs()
                    if vectors_only and target_preexisting and not root_is_file:
                        diff = await SemanticProcessor(
                            vlm_resolver=self.vlm_resolver
                        )._sync_topdown_recursive(temp_uri, root_uri, ctx=ctx, lock=resource_lock)
                        sync_deleted_files = list(getattr(diff, "deleted_files", []))
                        sync_deleted_dirs = list(getattr(diff, "deleted_dirs", []))
                    else:
                        await viking_fs.persist_temp_tree(
                            temp_uri, root_uri, ctx=ctx, lease_ref=resource_lock
                        )
                    if not root_is_file:
                        await rewrite_image_uris(
                            root_uri,
                            ctx=ctx,
                            lease_ref=resource_lock,
                        )
                    if temp_dir_path:
                        await viking_fs.delete_temp(temp_dir_path, ctx=ctx)
                if (
                    context_update_plan is None
                    and vectors_only
                    and (sync_deleted_files or sync_deleted_dirs)
                ):
                    await self._delete_removed_resource_vectors(
                        files=sync_deleted_files, dirs=sync_deleted_dirs, ctx=ctx
                    )
                if should_refresh_file_parent:
                    await (await self._summarizer_for(ctx)).refresh_file_parent(
                        file_uri=file_refresh.file_uri,
                        ctx=ctx,
                        skip_vectorization=not build_index,
                        ingest_options=ingest_options,
                        created=not target_preexisting,
                        file_md5=file_refresh.md5,
                        file_abstract="",
                    )
                elif build_index and context_update_plan is None:
                    if root_is_file:
                        with bind_ingestion_source_type(source_type):
                            await self._vectorize_resource_file(
                                root_uri,
                                ctx=ctx,
                                ingest_options=ingest_options,
                                file_md5=(prepared.get("file_md5s") or {}).get(root_uri),
                            )
                    elif vectors_only:
                        with bind_ingestion_source_type(source_type):
                            await self._vectorize_resource_files(
                                root_uri, ctx=ctx, ingest_options=ingest_options
                            )
            except BaseException:
                await cleanup_artifact_if_owned()
                raise
            finally:
                await get_viking_fs()._async_agfs.pathlock_release(resource_lock)
        elif should_refresh_file_parent:
            try:
                await (await self._summarizer_for(ctx)).refresh_file_parent(
                    file_uri=file_refresh.file_uri,
                    ctx=ctx,
                    skip_vectorization=not build_index,
                    ingest_options=ingest_options,
                    created=not target_preexisting,
                    file_md5=file_refresh.md5,
                    file_abstract="",
                )
            except BaseException:
                await cleanup_artifact_if_owned()
                raise
        elif build_index and context_update_plan is None and root_is_file:
            try:
                with bind_ingestion_source_type(source_type):
                    await self._vectorize_resource_file(
                        root_uri,
                        ctx=ctx,
                        ingest_options=ingest_options,
                        file_md5=(prepared.get("file_md5s") or {}).get(root_uri),
                    )
            except BaseException:
                await cleanup_artifact_if_owned()
                raise
        elif build_index and context_update_plan is None and vectors_only:
            try:
                artifact_store = (
                    self._build_parse_output_store() if artifact_ref is not None else None
                )
                with bind_ingestion_source_type(source_type):
                    await self._vectorize_resource_files(
                        root_uri,
                        ctx=ctx,
                        ingest_options=ingest_options,
                        artifact_store=artifact_store,
                        artifact_ref=artifact_ref,
                        artifact_files=prepared.get("artifact_files"),
                        file_md5s=prepared.get("file_md5s"),
                    )
            except BaseException:
                await cleanup_artifact_if_owned()
                raise
        if ingest_options.acl_update:
            await get_viking_fs().acl_manager.apply_indexed_update(ingest_options.acl_update, ctx)
        await cleanup_artifact_if_owned()
        return result

    @staticmethod
    def _semantic_source_metadata(
        *,
        path: str,
        prepared_resource: Optional["LocalResource"],
        source_format: Optional[str],
    ) -> Dict[str, str]:
        """Return the stable origin metadata carried only by the import root."""

        if prepared_resource is not None:
            return {
                "kind": str(prepared_resource.source_type),
                "uri": str(prepared_resource.original_source),
            }
        if source_format == "repository":
            kind = "git"
        elif path.startswith(("http://", "https://")):
            kind = "http"
        elif path.startswith(("git@", "ssh://", "git://")):
            kind = "git"
        else:
            kind = "local"
        return {"kind": kind, "uri": str(path)}

    async def _enqueue_index_actions(
        self,
        actions: Any,
        *,
        ctx: RequestContext,
        ingest_options: IngestOptions | None = None,
        source_contents: Optional[Dict[tuple[str, int], str | bytes]] = None,
        source_metadata: Optional[Dict[str, Any]] = None,
    ) -> int:
        from collections import Counter

        from openviking.core.skill_loader import SkillLoader
        from openviking.storage.index_action import IndexAction
        from openviking.storage.queuefs import get_queue_manager
        from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
        from openviking.telemetry import get_current_telemetry
        from openviking.utils.embedding_utils import _enqueue_embedding_message

        if source_contents is not None:
            missing_sources = [
                (action.uri, action.level)
                for action in actions
                if action.action in {IndexAction.UPSERT, IndexAction.MERGE}
                and (action.uri, action.level) not in source_contents
            ]
            if missing_sources:
                uri, level = missing_sources[0]
                source_name = (
                    f"directory L{level}"
                    if level in {int(ContextLevel.ABSTRACT), int(ContextLevel.OVERVIEW)}
                    else "file"
                )
                raise ValueError(f"{source_name} vector source is unavailable: {uri}")

        queue_manager = get_queue_manager()
        embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING, allow_create=True)
        telemetry_id = get_current_telemetry().telemetry_id
        action_counts = Counter(action.action.value for action in actions)
        completed_actions = 0
        delete_ids = [action.record_id for action in actions if action.action == IndexAction.DELETE]
        if delete_ids:
            message = EmbeddingMsg.for_delete(
                record_ids=delete_ids,
                context_data={
                    "uri": actions[0].uri,
                    "account_id": ctx.account_id,
                    "owner_user_id": ctx.user.user_id,
                },
                telemetry_id=telemetry_id,
            )
            await _enqueue_embedding_message(
                embedding_queue,
                message,
            )
            completed_actions += len(delete_ids)
        directory_actions: dict[str, dict[int, Any]] = {}
        for action in actions:
            if action.action in {IndexAction.UPSERT, IndexAction.MERGE}:
                if action.level in {
                    int(ContextLevel.ABSTRACT),
                    int(ContextLevel.OVERVIEW),
                }:
                    directory_actions.setdefault(action.uri, {})[action.level] = action
                    continue
                file_kwargs = {
                    "ctx": ctx,
                    "file_md5": action.md5,
                    "ingest_options": ingest_options,
                    "scalar_override": {
                        **dict(action.upsert_fields),
                        "_record_id": action.record_id,
                    },
                    "action": action.action.value,
                    "field_patch": action.field_patch,
                    "summary": action.summary,
                }
                source = (source_contents or {}).get((action.uri, action.level))
                if source is not None:
                    if not isinstance(source, bytes):
                        raise ValueError(f"file vector source must be bytes: {action.uri}")
                    file_kwargs["file_content"] = source
                    if getattr(self.vikingdb, "uses_content_field", False):
                        file_kwargs["materialize_content"] = True
                if await self._vectorize_resource_file(action.uri, **file_kwargs):
                    completed_actions += 1
                continue
            if action.action != IndexAction.UPDATE_FIELDS:
                continue
            assert action.field_patch is not None
            message = EmbeddingMsg.for_update_fields(
                record_id=action.record_id,
                field_patch=action.field_patch,
                context_data={
                    "uri": action.uri,
                    "level": action.level,
                    "account_id": ctx.account_id,
                    "owner_user_id": ctx.user.user_id,
                },
                telemetry_id=telemetry_id,
            )
            if await _enqueue_embedding_message(
                embedding_queue,
                message,
            ):
                completed_actions += 1
        for uri, levels in directory_actions.items():
            abstract = (source_contents or {}).get((uri, int(ContextLevel.ABSTRACT)), "")
            overview = (source_contents or {}).get((uri, int(ContextLevel.OVERVIEW)), "")
            if isinstance(abstract, bytes):
                abstract = abstract.decode("utf-8")
            if isinstance(overview, bytes):
                overview = overview.decode("utf-8")
            if int(ContextLevel.ABSTRACT) in levels and not isinstance(abstract, str):
                raise ValueError(f"directory L0 vector source is unavailable: {uri}")
            if int(ContextLevel.OVERVIEW) in levels and not isinstance(overview, str):
                raise ValueError(f"directory L1 vector source is unavailable: {uri}")
            meta = None
            if context_type_for_uri(uri) == "skill" and abstract:
                try:
                    skill = SkillLoader.parse(str(abstract))
                except ValueError:
                    skill = None
                if skill is not None:
                    meta = {
                        "name": skill["name"],
                        "description": skill.get("description", ""),
                        "tags": skill.get("tags", []),
                        "allowed_tools": skill.get("allowed_tools", []),
                    }
                else:
                    try:
                        import yaml

                        parsed = yaml.safe_load(str(abstract))
                    except Exception:
                        parsed = None
                    if isinstance(parsed, dict):
                        meta = {
                            key: parsed.get(key, [] if key in {"tags", "allowed_tools"} else "")
                            for key in ("name", "description", "tags", "allowed_tools")
                        }
                if meta is not None and source_metadata is not None:
                    meta["source_path"] = str(source_metadata.get("uri") or "")
            enqueued_levels = await vectorize_directory_meta(
                uri,
                abstract,
                overview,
                context_type=context_type_for_uri(uri),
                ctx=ctx,
                include_abstract=int(ContextLevel.ABSTRACT) in levels,
                include_overview=int(ContextLevel.OVERVIEW) in levels,
                content_is_body=True,
                actions={level: action.action.value for level, action in levels.items()},
                scalar_overrides={
                    level: {
                        **dict(action.upsert_fields),
                        "_record_id": action.record_id,
                    }
                    for level, action in levels.items()
                },
                field_patches={
                    level: action.field_patch
                    for level, action in levels.items()
                    if action.field_patch is not None
                },
                md5s={
                    level: action.md5 for level, action in levels.items() if action.md5 is not None
                },
                **({"meta": meta} if meta is not None else {}),
            )
            completed_actions += len(enqueued_levels)
        logger.debug(
            "[DirectIndexActions] %s root=%s action_counts=%s action_count=%d",
            log_correlation(),
            actions[0].uri if actions else "",
            dict(action_counts),
            len(actions),
        )
        return completed_actions

    async def _delete_removed_resource_vectors(
        self,
        *,
        files: list[str],
        dirs: list[str],
        ctx: RequestContext,
    ) -> None:
        for uri in dict.fromkeys(files):
            records = await self.vikingdb.get_context_by_uri(
                uri=uri,
                level=int(ContextLevel.DETAIL),
                limit=100,
                ctx=ctx,
            )
            ids = [str(record["id"]) for record in records if record.get("id")]
            if ids:
                await self.vikingdb.delete(ids, ctx=ctx)
        for uri in dict.fromkeys(dirs):
            records = await self.vikingdb.filter(
                filter=And(
                    [
                        PathScope("uri", uri, depth=-1),
                        Eq("level", int(ContextLevel.DETAIL)),
                        Eq("account_id", ctx.account_id),
                    ]
                ),
                limit=VECTORDB_MAX_QUERY_LIMIT,
                output_fields=["id"],
                ctx=ctx,
            )
            ids = [str(record["id"]) for record in records if record.get("id")]
            if ids:
                await self.vikingdb.delete(ids, ctx=ctx)

    async def _vectorize_resource_files(
        self,
        root_uri: str,
        *,
        ctx: RequestContext,
        ingest_options: IngestOptions | None = None,
        artifact_store: Any = None,
        artifact_ref: Any = None,
        artifact_files: Optional[List[str]] = None,
        file_md5s: Optional[Dict[str, str]] = None,
    ) -> None:
        ingest_options = IngestOptions.from_value(ingest_options)
        viking_fs = get_viking_fs()
        files: list[tuple[str, str, str]] = []
        if artifact_files is not None:
            for rel_path in artifact_files:
                entry_uri = VikingURI(root_uri).join(rel_path).uri
                parent = VikingURI(entry_uri).parent
                if parent is not None:
                    files.append((entry_uri, rel_path.rsplit("/", 1)[-1], parent.uri))
        else:
            entries = await viking_fs.tree(
                root_uri,
                node_limit=None,
                level_limit=None,
                ctx=ctx,
            )
            for entry in entries:
                entry_uri = entry.get("uri") if isinstance(entry, dict) else None
                if not entry_uri or entry.get("isDir"):
                    continue
                name = entry.get("name") or entry_uri.rsplit("/", 1)[-1]
                if str(name).startswith("."):
                    continue
                parent = VikingURI(entry_uri).parent
                if parent is None:
                    continue
                files.append((entry_uri, str(name), parent.uri))

        config = get_openviking_config().queue_workers.add_resource
        concurrency = max(
            1,
            min(
                int(config.file_vectorization_concurrency),
                _MAX_FILE_VECTORIZATION_CONCURRENCY,
            ),
        )

        async def vectorize(entry_uri: str, name: str, parent_uri: str) -> None:
            file_content = None
            if artifact_store is not None and artifact_ref is not None:
                rel_path = entry_uri[len(root_uri.rstrip("/")) + 1 :]
                artifact_rel = (
                    f"{artifact_ref.resource_rel.strip('/')}/{rel_path}"
                    if artifact_ref.resource_rel
                    else rel_path
                )
                file_content = await artifact_store.read_bytes(artifact_ref, artifact_rel)
            await vectorize_file(
                file_path=entry_uri,
                summary_dict={"name": name, "summary": ""},
                parent_uri=parent_uri,
                context_type=context_type_for_uri(entry_uri),
                ctx=ctx,
                ingest_options=ingest_options,
                file_md5=(file_md5s or {}).get(entry_uri),
                file_content=file_content,
            )

        for start in range(0, len(files), concurrency):
            tasks = [
                asyncio.create_task(vectorize(entry_uri, name, parent_uri))
                for entry_uri, name, parent_uri in files[start : start + concurrency]
            ]
            try:
                await asyncio.gather(*tasks)
            except BaseException:
                for task in tasks:
                    task.cancel()
                await asyncio.gather(*tasks, return_exceptions=True)
                raise

    async def _vectorize_resource_file(
        self,
        file_uri: str,
        *,
        ctx: RequestContext,
        ingest_options: IngestOptions | None = None,
        file_md5: str | None = None,
        scalar_override: Optional[Dict[str, Any]] = None,
        field_patch: FieldPatch | None = None,
        action: str = "merge",
        file_content: bytes | None = None,
        materialize_content: bool = False,
        summary: str = "",
    ) -> bool:
        parent = VikingURI(file_uri).parent
        if parent is None:
            return False
        name = file_uri.rsplit("/", 1)[-1]
        return await vectorize_file(
            file_path=file_uri,
            summary_dict={"name": name, "summary": summary},
            parent_uri=parent.uri,
            context_type=context_type_for_uri(file_uri),
            ctx=ctx,
            ingest_options=IngestOptions.from_value(ingest_options),
            file_md5=file_md5,
            scalar_override=scalar_override,
            field_patch=field_patch,
            action=action,
            file_content=file_content,
            materialize_content=materialize_content,
        )

    async def reserve_unique_candidate(
        self,
        *,
        candidate_uri: str,
        ctx: RequestContext,
        max_attempts: int = 100,
        root_is_file: bool = False,
        target_validator: Optional[Callable[[str, RequestContext], Awaitable[None]]] = None,
    ) -> tuple[str, Dict[str, Any]]:
        """Validate the first free candidate before reserving it with a type-aware lock."""
        from openviking.storage.errors import ResourceBusyError

        viking_fs = get_viking_fs()
        last_busy_error: Optional[ResourceBusyError] = None
        await self.ensure_candidate_parent_write_access(candidate_uri=candidate_uri, ctx=ctx)

        for attempt in range(max_attempts + 1):
            root_uri = candidate_uri if attempt == 0 else f"{candidate_uri}_{attempt}"
            if await viking_fs.exists(root_uri, ctx=ctx):
                continue

            if target_validator is not None:
                await target_validator(root_uri, ctx)

            dst_path = viking_fs._uri_to_path(root_uri, ctx=ctx)
            try:
                resource_lock = await self.acquire_resource_lock(
                    dst_path,
                    uri=root_uri,
                    timeout=0.0,
                    root_is_file=root_is_file,
                )
                return root_uri, resource_lock
            except ResourceBusyError as exc:
                last_busy_error = exc
                continue

        if last_busy_error is not None:
            raise ResourceBusyError(
                f"All auto-named candidates are temporarily busy for {candidate_uri} "
                f"after checking {max_attempts + 1} candidates",
                uri=candidate_uri,
                conflict_type="auto_name_reservation_busy",
                retryable=True,
            ) from last_busy_error

        raise FileExistsError(
            f"Cannot resolve unique name for {candidate_uri} after {max_attempts} attempts"
        )

    async def ensure_candidate_parent_write_access(
        self,
        *,
        candidate_uri: str,
        ctx: RequestContext,
    ) -> None:
        """Require create permission for an auto-named resource candidate."""
        parent_uri = VikingURI(candidate_uri).parent
        if parent_uri is None:
            raise ValueError(f"Resource candidate must have a parent: {candidate_uri}")
        await get_viking_fs()._ensure_access(
            parent_uri.uri,
            ctx,
            action=AclAction.WRITE,
        )

    @staticmethod
    async def acquire_resource_lock(
        path: str,
        *,
        uri: str = "",
        timeout: float = 0.0,
        root_is_file: bool = False,
    ) -> Dict[str, Any]:
        """Acquire a file-exact or directory-tree resource lock."""
        from openviking.storage.errors import ResourceBusyError

        try:
            pathlock = get_viking_fs()._async_agfs
            acquire = (
                pathlock.pathlock_acquire_exact if root_is_file else pathlock.pathlock_acquire_tree
            )
            return await acquire(path, timeout_secs=timeout)
        except LockAcquisitionError as exc:
            logger.warning(f"[ResourceProcessor] Failed to acquire resource lock on {path}")
            raise ResourceBusyError(
                f"Resource is busy: {uri or path}",
                uri=uri or path,
                conflict_type="path_busy",
                retryable=True,
            ) from exc