Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

85.3 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""SemanticProcessor: Processes messages from SemanticQueue, generates .abstract.md and .overview.md."""

import asyncio
import re
import threading
import time
from contextlib import nullcontext
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple

from openviking.core.namespace import classify_uri
from openviking.models.model_error import ModelErrorKind, model_error_kind
from openviking.observability.context import (
    bind_root_observability_context,
    reset_root_observability_context,
)
from openviking.parse.image_rewrite import (
    IMAGE_MAPPINGS_FILENAME,
    rewrite_image_uris,
)
from openviking.parse.parsers.constants import (
    CODE_EXTENSIONS,
    DOCUMENTATION_EXTENSIONS,
    FILE_TYPE_CODE,
    FILE_TYPE_DOCUMENTATION,
    FILE_TYPE_OTHER,
)
from openviking.parse.parsers.media.utils import (
    MPEG_TS_PROBE_BYTES,
    generate_audio_summary,
    generate_image_summary,
    generate_video_summary,
    get_media_type,
)
from openviking.prompts import render_prompt
from openviking.pyagfs.exceptions import AGFSNotADirectoryError
from openviking.server.identity import RequestContext, Role
from openviking.service.task_tracker import get_task_tracker, run_tracked_operation
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.service.task_work_index import detach_task_context, get_task_context
from openviking.storage.abstract_overview import (
    AbstractOverviewWriteResult,
    body_for_preview,
    freshness_metadata,
    markdown_safe_viking_uri,
    parse_overview_file_summaries,
    plan_abstract_overview_refresh,
    write_abstract_overview,
)
from openviking.storage.errors import (
    HandoffRecoveryBlocked,
    InvalidPathLockRequestError,
    LockAcquisitionError,
)
from openviking.storage.index_action import FieldPatch
from openviking.storage.pathlock_recovery import prepare_handoff_retry
from openviking.storage.pathlock_transfer import PathLockGuard
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
from openviking.storage.queuefs.process_result import ProcessResult
from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor, SemanticTreeStats
from openviking.storage.queuefs.semantic_lock import SemanticLockScope
from openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_coalesce_key
from openviking.storage.queuefs.semantic_ops.freshness_policy import FreshnessAction
from openviking.storage.queuefs.semantic_queue import is_semantic_msg_stale
from openviking.storage.queuefs.semantic_work import SemanticMessageWork, SkillSemanticMessageWork
from openviking.storage.viking_fs import LS_ALL_NODES, SyncDiff, get_viking_fs
from openviking.telemetry import (
    bind_telemetry,
    bind_telemetry_stage,
)
from openviking.telemetry.span_models import create_root_span_attributes
from openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen
from openviking.utils.ingest_options import IngestOptions
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils import VikingURI
from openviking_cli.utils.config import get_openviking_config
from openviking_cli.utils.logger import get_logger

if TYPE_CHECKING:
    from openviking.config.vlm import VLMHandle, VLMResolver

logger = get_logger(__name__)

_PERMANENT_STORAGE_ERRORS = (
    FileNotFoundError,
    PermissionError,
    IsADirectoryError,
    NotADirectoryError,
    AGFSNotADirectoryError,
    InvalidPathLockRequestError,
)


def _is_permanent_storage_error(error: BaseException) -> bool:
    return any(isinstance(item, _PERMANENT_STORAGE_ERRORS) for item in (error, error.__cause__))


def _executor_lock(msg: SemanticMsg, lock: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
    """Reuse content-write ownership only when it covers semantic sidecars."""
    if msg.generation_trigger != "content_write" or msg.lock_handoff is None:
        return lock
    covered_paths = msg.lock_handoff.get("covered_paths")
    if isinstance(covered_paths, list) and any(
        isinstance(item, dict) and item.get("kind") == "tree" for item in covered_paths
    ):
        return lock
    return None


class RequestQueueStats:
    processed: int = 0
    requeue_count: int = 0
    error_count: int = 0


class SemanticProcessor(DequeueHandlerBase):
    """
    Semantic processor, generates .abstract.md and .overview.md bottom-up.

    Processing flow:
    1. Concurrently generate summaries for files in directory
    2. Collect .abstract.md from subdirectories
    3. Generate .abstract.md and .overview.md for this directory
    4. Enqueue to EmbeddingQueue for vectorization
    """

    _stats_lock = threading.Lock()

    @staticmethod
    async def _cleanup_local_artifact(msg: SemanticMsg) -> None:
        try:
            if not msg.artifact_ref:
                return
            from openviking.parse.output import ParseArtifactRef, store_for_artifact_ref

            artifact_ref = ParseArtifactRef.from_dict(msg.artifact_ref)
            if artifact_ref.backend != "local":
                return
            store = store_for_artifact_ref(artifact_ref)
            await store.cleanup(artifact_ref)
        except Exception as exc:
            logger.warning("Failed to clean local parse artifact: %s", exc)

    _tree_stats_by_telemetry_id: Dict[str, SemanticTreeStats] = {}
    _tree_stats_by_uri: Dict[str, SemanticTreeStats] = {}
    _tree_stats_order: List[Tuple[str, str]] = []
    _request_stats_by_telemetry_id: Dict[str, RequestQueueStats] = {}
    _request_stats_order: List[str] = []
    _max_cached_stats = 256

    def __init__(
        self,
        max_concurrent_llm: int = 32,
        *,
        embedding_worker_stopped: Optional[Callable[[], bool]] = None,
        vlm_resolver: Optional["VLMResolver"] = None,
    ):
        """
        Initialize SemanticProcessor.

        Args:
            max_concurrent_llm: Maximum concurrent LLM calls
        """
        self.max_concurrent_llm = max_concurrent_llm
        self._default_ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
        self._circuit_breakers: Dict[str, CircuitBreaker] = {}
        self._breaker_lock = threading.Lock()
        self._embedding_worker_stopped = embedding_worker_stopped
        self._vlm_resolver = vlm_resolver

    def _account_breaker(self, account_id: str) -> CircuitBreaker:
        with self._breaker_lock:
            breaker = self._circuit_breakers.get(account_id)
            if breaker is None:
                breaker = self._circuit_breakers[account_id] = CircuitBreaker()
            return breaker

    async def _get_vlm_config(self, ctx: RequestContext) -> "VLMHandle":
        if self._vlm_resolver is None:
            raise RuntimeError("SemanticProcessor requires a VLM resolver for account-owned work")
        return await self._vlm_resolver.get_vlm(ctx.account_id)

    @classmethod
    def _cache_tree_stats(cls, telemetry_id: str, uri: str, stats: SemanticTreeStats) -> None:
        with cls._stats_lock:
            if telemetry_id:
                previous = cls._tree_stats_by_telemetry_id.get(telemetry_id)
                if previous is not None:
                    stats = SemanticTreeStats(
                        total_nodes=previous.total_nodes + stats.total_nodes,
                        pending_nodes=previous.pending_nodes + stats.pending_nodes,
                        in_progress_nodes=previous.in_progress_nodes + stats.in_progress_nodes,
                        done_nodes=previous.done_nodes + stats.done_nodes,
                        failures=[*previous.failures, *stats.failures],
                        indexed_records=previous.indexed_records + stats.indexed_records,
                    )
                cls._tree_stats_by_telemetry_id[telemetry_id] = stats
            cls._tree_stats_by_uri[uri] = stats
            cls._tree_stats_order.append((telemetry_id, uri))
            if len(cls._tree_stats_order) > cls._max_cached_stats:
                old_telemetry_id, old_uri = cls._tree_stats_order.pop(0)
                if old_telemetry_id:
                    cls._tree_stats_by_telemetry_id.pop(old_telemetry_id, None)
                cls._tree_stats_by_uri.pop(old_uri, None)

    @classmethod
    def consume_tree_stats(
        cls,
        telemetry_id: str = "",
        uri: Optional[str] = None,
    ) -> Optional[SemanticTreeStats]:
        with cls._stats_lock:
            if telemetry_id and telemetry_id in cls._tree_stats_by_telemetry_id:
                stats = cls._tree_stats_by_telemetry_id.pop(telemetry_id, None)
                if uri:
                    cls._tree_stats_by_uri.pop(uri, None)
                return stats
            if uri and uri in cls._tree_stats_by_uri:
                return cls._tree_stats_by_uri.pop(uri, None)
        return None

    @classmethod
    def _merge_request_stats(
        cls,
        telemetry_id: str,
        processed: int = 0,
        requeue_count: int = 0,
        error_count: int = 0,
    ) -> None:
        if not telemetry_id:
            return
        with cls._stats_lock:
            stats = cls._request_stats_by_telemetry_id.setdefault(telemetry_id, RequestQueueStats())
            stats.processed += processed
            stats.requeue_count += requeue_count
            stats.error_count += error_count
            cls._request_stats_order.append(telemetry_id)
            if len(cls._request_stats_order) > cls._max_cached_stats:
                old_telemetry_id = cls._request_stats_order.pop(0)
                if old_telemetry_id != telemetry_id:
                    cls._request_stats_by_telemetry_id.pop(old_telemetry_id, None)

    @classmethod
    def consume_request_stats(cls, telemetry_id: str) -> Optional[RequestQueueStats]:
        if not telemetry_id:
            return None
        with cls._stats_lock:
            return cls._request_stats_by_telemetry_id.pop(telemetry_id, None)

    @staticmethod
    def _ctx_from_semantic_msg(msg: SemanticMsg) -> RequestContext:
        return RequestContext(
            user=UserIdentifier(msg.account_id, msg.user_id),
            role=Role(msg.role),
            group_ids=tuple(msg.group_ids),
            bypass_acl=True,
        )

    def _detect_file_type(self, file_name: str) -> str:
        """
        Detect file type for summary prompt selection.

        Args:
            file_name: File name with extension

        Returns:
            FILE_TYPE_CODE, FILE_TYPE_DOCUMENTATION, or FILE_TYPE_OTHER
        """
        file_name_lower = file_name.lower()

        # Documentation prompts should win over broad code skeleton recognition.
        for ext in DOCUMENTATION_EXTENSIONS:
            if file_name_lower.endswith(ext):
                return FILE_TYPE_DOCUMENTATION

        from openviking.parse.parsers.code.ast.providers import supports_code_skeleton

        if supports_code_skeleton(file_name):
            return FILE_TYPE_CODE

        # Keep legacy extension-based routing for code-like text formats not
        # covered by tags queries or tree-sitter-language-pack.
        for ext in CODE_EXTENSIONS:
            if file_name_lower.endswith(ext):
                return FILE_TYPE_CODE

        # Default to other
        return FILE_TYPE_OTHER

    async def _reenqueue_semantic_msg(
        self,
        msg: SemanticMsg,
        *,
        enqueue: Optional[Callable[[Any, SemanticMsg], Awaitable[None]]] = None,
    ) -> None:
        """Re-enqueue a semantic message for later processing.

        Throttles with a sleep when the circuit breaker is open to prevent
        re-enqueue storms (messages cycling at 5/sec during OPEN window).
        """
        import asyncio

        from openviking.storage.queuefs import get_queue_manager

        # Throttle to prevent re-enqueue storm during OPEN window
        wait = self._account_breaker(msg.account_id).retry_after
        if wait > 0:
            await asyncio.sleep(wait)

        queue_manager = get_queue_manager()
        if queue_manager is not None:
            semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC)
            if enqueue is None:
                await semantic_queue.enqueue(msg)
            else:
                await enqueue(semantic_queue, msg)
            logger.info(f"Re-enqueued semantic message: {msg.uri}")
        else:
            logger.warning(f"No queue manager available, cannot re-enqueue: {msg.uri}")

    async def _enqueue_semantic_retry(
        self, queue, msg: SemanticMsg, scope: SemanticLockScope
    ) -> None:
        """Transfer the live semantic lease to a retry before releasing this worker.

        Reusing the consumed handoff would require acquiring an unrelated lock,
        which conflicts with an update request still waiting under its outer lease.
        """
        agfs = get_viking_fs()._async_agfs
        lock_guard = PathLockGuard(agfs, scope.lock)
        handoff = await lock_guard.to_handoff()
        msg.lock_handoff = handoff

        def accept(result: Any) -> bool:
            if result == "deduplicated":
                return False
            scope.lock = None
            scope._owned = False
            return True

        try:
            enqueue_id = await lock_guard.handoff_and_dispatch(
                handoff,
                lambda: queue.enqueue(msg, deduplicate_memory=False),
                accepted=accept,
            )
            if enqueue_id == "deduplicated":
                raise RuntimeError("Semantic retry was unexpectedly deduplicated")
        finally:
            if lock_guard.lease is not None:
                scope.lock = lock_guard.lease
                scope._owned = True

    async def _enqueue_skill_retry(self, queue, msg: SemanticMsg, scope: SemanticLockScope) -> None:
        """Compatibility wrapper for existing skill retry callers."""
        await self._enqueue_semantic_retry(queue, msg, scope)

    async def _requeue_semantic_msg_after_error(
        self,
        msg: SemanticMsg,
        error: Exception,
        *,
        work: SemanticMessageWork,
    ) -> ProcessResult:
        if isinstance(error, HandoffRecoveryBlocked):
            payload = await prepare_handoff_retry(msg, error, queue="Semantic")
            if payload is None:
                self._merge_request_stats(msg.telemetry_id, error_count=1)
                await self._cleanup_local_artifact(msg)
                return ProcessResult.failed(f"Pathlock recovery retry limit reached: {error}")
            msg.lock_handoff_retry = payload["lock_handoff_retry"]
        try:
            await work.reenqueue_after_error()
            self._merge_request_stats(msg.telemetry_id, requeue_count=1)
        except Exception as requeue_err:
            if isinstance(error, HandoffRecoveryBlocked):
                # No replacement is durable: propagate so the original delivery
                # remains recoverable rather than being ACKed as failed.
                raise
            logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")
            self._merge_request_stats(msg.telemetry_id, error_count=1)
            await self._cleanup_local_artifact(msg)
            return ProcessResult.failed(str(error))
        return ProcessResult.requeued()

    async def _enqueue_parent_refresh(
        self, msg: SemanticMsg, uri: str, *, l0_body_changed: bool
    ) -> None:
        if msg.generation_trigger == "content_copy":
            return
        if msg.context_type not in {"resource", "skill"}:
            return
        if not msg.propagate_to_parent:
            return
        parent = VikingURI(uri).parent
        if parent is None:
            return
        parent_uri = parent.uri.rstrip("/")
        if msg.context_type == "skill":
            classification = classify_uri(parent_uri)
            if (
                not classification.is_skill
                or classification.is_skill_namespace
                or classification.is_skill_root
            ):
                return
        if (
            not parent_uri
            or parent_uri in {"viking://", "viking:", "viking://user", "viking://agent"}
            or parent_uri == uri.rstrip("/")
        ):
            return
        parent_ctx = self._ctx_from_semantic_msg(msg)
        semantic_config = get_openviking_config().semantic
        try:
            decision = await plan_abstract_overview_refresh(
                viking_fs=get_viking_fs(),
                dir_uri=parent_uri,
                changed_entries=1,
                ctx=parent_ctx,
                l0_body_changed=l0_body_changed,
                # This helper handles automatic upward propagation only. Manual
                # refresh/ingest bypasses the threshold for its requested root,
                # not for every ancestor reached afterwards.
                force_refresh=False,
                overview_sample_limit=getattr(semantic_config, "overview_sample_limit", 32),
                refresh_ratio=getattr(semantic_config, "freshness_refresh_ratio", 0.10),
                lock_timeout_secs=1.0,
            )
        except LockAcquisitionError:
            logger.info(
                "Skipping best-effort parent freshness update because sidecars are busy: %s",
                parent_uri,
            )
            return

        if decision.action is not FreshnessAction.REFRESH_NOW:
            logger.debug(
                "Parent semantic refresh %s for %s (pending=%d, total=%d)",
                decision.action.value,
                parent_uri,
                decision.pending_after,
                decision.total_entries,
            )
            return

        from openviking.storage.queuefs import get_queue_manager

        queue_manager = get_queue_manager()
        semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
        parent_msg = SemanticMsg(
            uri=parent_uri,
            context_type=msg.context_type,
            recursive=False,
            account_id=msg.account_id,
            user_id=msg.user_id,
            group_ids=msg.group_ids,
            peer_id=msg.peer_id,
            role=msg.role,
            skip_vectorization=msg.skip_vectorization,
            changes={"modified": [uri]},
            generation_trigger="parent_refresh",
            coalesce_key=build_semantic_coalesce_key(
                context_type=msg.context_type,
                uri=parent_uri,
                account_id=msg.account_id,
                user_id=msg.user_id,
                peer_id=msg.peer_id,
            ),
        )
        with detach_task_context():
            await semantic_queue.enqueue(parent_msg)
        logger.info("Enqueued parent semantic refresh: %s", parent_uri)

    def _message_work(
        self, msg: SemanticMsg, lock: Optional[Dict[str, Any]] = None
    ) -> SemanticMessageWork:
        work_type = SkillSemanticMessageWork if msg.context_type == "skill" else SemanticMessageWork
        return work_type(self, msg, lock)

    async def on_dequeue(
        self,
        data: Optional[Dict[str, Any]],
        lock: Optional[Dict[str, Any]] = None,
    ) -> ProcessResult:
        """Process dequeued SemanticMsg, recursively process all subdirectories."""
        import json

        if not data:
            return ProcessResult.success()
        try:
            if "data" in data and isinstance(data["data"], str):
                data = json.loads(data["data"])
            msg = SemanticMsg.from_dict(data)
        except Exception as exc:
            logger.error("Invalid semantic message: %s", exc, exc_info=True)
            return ProcessResult.failed(str(exc))

        if msg.context_type != "skill" or get_task_context() is not None:
            return await self._process_message(msg, lock)

        # Parent refreshes and recovered taskless Skill messages need their own
        # child-work collector so package ownership lasts through vector writes.
        result: Optional[ProcessResult] = None

        async def process() -> Dict[str, Any]:
            nonlocal result
            result = await self._process_message(msg, lock)
            if result.error is not None:
                context = get_task_context()
                assert context is not None
                await get_task_tracker().fail(
                    context.task_id,
                    result.error,
                    account_id=msg.account_id,
                    user_id=msg.user_id,
                )
            return {"uri": msg.uri, "outcome": result.outcome.value}

        await run_tracked_operation(
            "semantic_refresh",
            account_id=msg.account_id,
            user_id=msg.user_id,
            resource_id=msg.uri,
            operation=process,
            wait=False,
        )
        assert result is not None
        return result

    async def _process_message(
        self,
        msg: SemanticMsg,
        lock: Optional[Dict[str, Any]],
    ) -> ProcessResult:
        collector = None
        work: Optional[SemanticMessageWork] = None
        execute_started_at: float | None = None
        queue_wait_ms = 0.0
        execute_status = "ok"
        breaker = None
        try:
            breaker = self._account_breaker(msg.account_id)
            work = self._message_work(msg, lock)
            work.start()
            execute_started_at = time.perf_counter()
            if msg.queue_enqueued_at > 0:
                queue_wait_ms = max((time.time() - msg.queue_enqueued_at) * 1000.0, 0.0)
            if VikingURI(msg.uri).parent is None:
                logger.warning("Skipping semantic generation for root URI: %s", msg.uri)
                await self._cleanup_local_artifact(msg)
                return ProcessResult.success()
            if is_semantic_msg_stale(msg):
                live_file_changes = {
                    kind: list(msg.changes.get(kind, []))
                    for kind in ("added", "modified")
                    if msg.changes and msg.changes.get(kind)
                }
                if msg.generation_trigger == "content_write" and live_file_changes:
                    # Coalescing replaces directory aggregation, not per-file
                    # maintenance. Let the newest message aggregate while this
                    # one still summarizes/vectorizes its changed files.
                    logger.info(
                        "Downgrading stale semantic message to file-only work: uri=%s version=%s",
                        msg.uri,
                        msg.coalesce_version,
                    )
                    msg.aggregate_directory = False
                    msg.changes = live_file_changes
                    msg.coalesce_key = ""
                    msg.coalesce_version = 0
                else:
                    logger.info(
                        "Skipping stale semantic message: uri=%s version=%s",
                        msg.uri,
                        msg.coalesce_version,
                    )
                    await work.skip()
                    await self._cleanup_local_artifact(msg)
                    return ProcessResult.success()
            # Circuit breaker: if API is known-broken, re-enqueue and wait
            try:
                breaker.check()
            except CircuitBreakerOpen:
                logger.warning(
                    f"Circuit breaker is open, re-enqueueing semantic message: {msg.uri}"
                )
                await work.reenqueue()
                self._merge_request_stats(msg.telemetry_id, requeue_count=1)
                return ProcessResult.requeued()
            collector = work.resolve_telemetry()
            telemetry_ctx = bind_telemetry(collector) if collector is not None else nullcontext()
            with telemetry_ctx:
                root_attrs = create_root_span_attributes(
                    http_method="QUEUE",
                    http_route=msg.context_type or "/queuefs/semantic",
                    request_id=msg.telemetry_id or msg.id,
                    url_path=msg.uri,
                )
                root_attrs.account_id = msg.account_id
                root_attrs.user_id = msg.user_id
                root_context_token = bind_root_observability_context(root_attrs)
                try:
                    current_ctx = self._ctx_from_semantic_msg(msg)
                    logger.info(
                        f"Processing semantic generation for: {msg.uri} (recursive={msg.recursive})"
                    )

                    logger.debug("Processing semantic message id=%s uri=%s", msg.id, msg.uri)

                    # Resolving a deleted root can recreate it for lock metadata.
                    # Settle queued ownership before acknowledging skipped work.
                    if not await get_viking_fs().exists(msg.uri, ctx=current_ctx):
                        logger.info("Skipping semantic message for missing root: uri=%s", msg.uri)
                        await work.skip()
                        return ProcessResult.success()

                    if self._vlm_resolver is None:
                        raise RuntimeError(
                            "SemanticProcessor requires a VLM resolver for account-owned work"
                        )
                    if not await work.acquire_lock(current_ctx):
                        return ProcessResult.success()
                    semantic_lock = work.scope
                    assert semantic_lock is not None
                    dag_stats = None
                    processing_succeeded = False
                    try:
                        if msg.plan is not None:
                            if msg.uri.rstrip("/") != msg.plan.root_uri:
                                raise ValueError("semantic message URI must match plan root_uri")
                            if msg.context_type != msg.plan.context_type:
                                raise ValueError(
                                    "semantic message context_type must match semantic plan"
                                )
                            for run_uri in msg.plan.execution_root_uris():
                                executor = SemanticTreeExecutor(
                                    processor=self,
                                    context_type=msg.context_type,
                                    max_concurrent_llm=self.max_concurrent_llm,
                                    ctx=current_ctx,
                                    lock=semantic_lock.lock,
                                    source=msg.plan.source_metadata,
                                    semantic_plan=msg.plan,
                                    telemetry_id=msg.telemetry_id,
                                    generation_trigger=msg.generation_trigger,
                                )
                                await executor.run(run_uri)
                                self._cache_tree_stats(
                                    msg.telemetry_id, run_uri, executor.get_stats()
                                )
                                if not executor.stale:
                                    write_result = getattr(
                                        executor,
                                        "root_write_result",
                                        AbstractOverviewWriteResult(
                                            wrote=True, abstract_body_changed=True
                                        ),
                                    )
                                    await self._enqueue_parent_refresh(
                                        msg,
                                        run_uri,
                                        l0_body_changed=write_result.abstract_body_changed,
                                    )
                            from collections import Counter

                            entries = msg.plan.tree.entries
                            action_counts = Counter(
                                entry.semantic_action.value for entry in entries
                            )
                            vector_slots = sum(
                                slot.action.value in {"upsert", "merge"}
                                for entry in entries
                                for slot in entry.index_slots
                            )
                            logger.debug(
                                "[SemanticPlanExecution] root=%s execution_roots=%d "
                                "semantic_entries=%d actions=%s planned_vector_upserts=%d",
                                msg.plan.root_uri,
                                len(msg.plan.execution_root_uris()),
                                len(entries),
                                dict(action_counts),
                                vector_slots,
                            )
                        # Regular memory writes keep their specialized update path.
                        # Callers must explicitly opt into directory aggregation; the
                        # trigger remains descriptive metadata, not an algorithm switch.
                        elif msg.context_type == "memory" and not msg.use_hierarchical_aggregation:
                            await self._process_memory_directory(
                                msg,
                                ctx=current_ctx,
                                lock=semantic_lock.lock,
                            )
                        else:
                            is_incremental = False
                            target_uri = msg.target_uri
                            run_uri = msg.uri
                            changes = msg.changes
                            viking_fs = get_viking_fs()
                            if msg.target_uri:
                                target_exists = await viking_fs.exists(
                                    msg.target_uri, ctx=current_ctx
                                )
                                if msg.uri != msg.target_uri:
                                    logger.info(
                                        "Syncing semantic source into target before processing: "
                                        f"{msg.uri} -> {msg.target_uri}"
                                    )
                                    diff = await work.run_write(
                                        lambda: self._sync_topdown_recursive(
                                            msg.uri,
                                            msg.target_uri,
                                            ctx=current_ctx,
                                            lock=semantic_lock.lock,
                                        )
                                    )
                                    logger.info(
                                        "[SyncDiff] Diff computed: "
                                        f"added_files={len(diff.added_files)}, "
                                        f"deleted_files={len(diff.deleted_files)}, "
                                        f"updated_files={len(diff.updated_files)}, "
                                        f"added_dirs={len(diff.added_dirs)}, "
                                        f"deleted_dirs={len(diff.deleted_dirs)}"
                                    )
                                    changes = diff.to_changes()
                                    is_incremental = True
                                    target_uri = msg.target_uri
                                    run_uri = msg.target_uri
                                elif (
                                    target_exists
                                    and msg.changes is not None
                                    and msg.uri == msg.target_uri
                                ):
                                    is_incremental = True
                                    logger.info(
                                        f"Using direct incremental semantic update for: {msg.uri}"
                                    )
                            elif msg.changes is not None:
                                is_incremental = True
                                target_uri = msg.uri
                                logger.info(
                                    f"Using direct incremental semantic update for: {msg.uri}"
                                )

                            executor = SemanticTreeExecutor(
                                processor=self,
                                context_type=msg.context_type,
                                max_concurrent_llm=self.max_concurrent_llm,
                                ctx=current_ctx,
                                incremental_update=is_incremental,
                                target_uri=target_uri,
                                recursive=msg.recursive,
                                lock=_executor_lock(msg, semantic_lock.lock),
                                is_code_repo=msg.is_code_repo,
                                changes=changes,
                                skip_vectorization=msg.skip_vectorization,
                                ingest_options=msg.ingest_options,
                                coalesce_key=msg.coalesce_key,
                                coalesce_version=msg.coalesce_version,
                                source=msg.source,
                                generation_trigger=msg.generation_trigger,
                                aggregate_directory=msg.aggregate_directory,
                                copy_source_uri=msg.copy_source_uri,
                                file_md5s=msg.file_md5s,
                                artifact_files=msg.artifact_files,
                                file_abstracts=msg.file_abstracts,
                                file_vector_actions=msg.file_vector_actions,
                                file_ingest_options=msg.file_ingest_options,
                                telemetry_id=msg.telemetry_id,
                            )
                            await executor.run(run_uri)
                            dag_stats = executor.get_stats()
                            self._cache_tree_stats(
                                msg.telemetry_id,
                                run_uri,
                                dag_stats,
                            )
                            if not executor.stale and msg.aggregate_directory:
                                write_result = getattr(
                                    executor,
                                    "root_write_result",
                                    AbstractOverviewWriteResult(
                                        wrote=True, abstract_body_changed=True
                                    ),
                                )
                                await self._enqueue_parent_refresh(
                                    msg,
                                    target_uri or msg.uri,
                                    l0_body_changed=write_result.abstract_body_changed,
                                )
                        processing_succeeded = True
                    finally:
                        await work.finish_processing(processing_succeeded)
                    failure = work.failure_result(dag_stats)
                    if failure is not None:
                        return failure
                    self._merge_request_stats(msg.telemetry_id, processed=1)
                    logger.info(f"Completed semantic generation for: {msg.uri}")
                    breaker.record_success()
                    await self._cleanup_local_artifact(msg)
                    return ProcessResult.success()
                finally:
                    reset_root_observability_context(root_context_token)

        except asyncio.CancelledError:
            if work is not None:
                await work.cancel()
            raise
        except Exception as e:
            if isinstance(e, LockAcquisitionError):
                execute_status = "requeued"
                logger.warning(
                    "Lock error processing semantic message, re-enqueueing without "
                    "tripping API circuit breaker: %s",
                    e,
                    exc_info=True,
                )
                if msg is not None and work is not None:
                    result = await self._requeue_semantic_msg_after_error(
                        msg,
                        e,
                        work=work,
                    )
                    if result.error is not None:
                        execute_status = "error"
                    return result
                return ProcessResult.failed(str(e))

            kind = model_error_kind(e)
            if kind is ModelErrorKind.INPUT_TOO_LARGE:
                execute_status = "error"
                logger.error(
                    f"Input too large processing semantic message, dropping: {e}",
                    exc_info=True,
                )
                if msg is not None:
                    self._merge_request_stats(msg.telemetry_id, error_count=1)
                if msg is not None:
                    await self._cleanup_local_artifact(msg)
                return ProcessResult.failed(str(e))
            elif _is_permanent_storage_error(e) or kind in {
                ModelErrorKind.MODEL_NOT_FOUND,
                ModelErrorKind.UNSUPPORTED,
                ModelErrorKind.INVALID_REQUEST,
                ModelErrorKind.CONTENT_SAFETY,
            }:
                execute_status = "error"
                logger.critical(
                    f"Permanent error processing semantic message, dropping: {e}",
                    exc_info=True,
                )
                if breaker is not None and kind is not None:
                    breaker.record_failure(e)
                if msg is not None:
                    self._merge_request_stats(msg.telemetry_id, error_count=1)
                if msg is not None:
                    await self._cleanup_local_artifact(msg)
                return ProcessResult.failed(str(e))
            else:
                execute_status = "error"
                if breaker is not None and kind is not None:
                    breaker.record_failure(e)
                logger.error(
                    f"Error processing semantic message, not re-enqueueing: {e}",
                    exc_info=True,
                )
                if msg is not None:
                    self._merge_request_stats(msg.telemetry_id, error_count=1)
                    await self._cleanup_local_artifact(msg)
                return ProcessResult.failed(str(e))
        finally:
            if msg is not None and execute_started_at is not None:
                from openviking.metrics.datasources.resource import ResourceIngestionEventDataSource

                ResourceIngestionEventDataSource.record_stage(
                    stage="semantic_queue_wait",
                    status=execute_status,
                    duration_seconds=queue_wait_ms / 1000.0,
                    account_id=msg.account_id,
                )
                ResourceIngestionEventDataSource.record_stage(
                    stage="semantic_execute",
                    status=execute_status,
                    duration_seconds=(time.perf_counter() - execute_started_at),
                    account_id=msg.account_id,
                )
            if work is not None:
                await work.close()

    async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
        """Release a queued semantic lock before cancelled work is ACKed."""
        try:
            import json

            payload = data.get("data", data) if isinstance(data, dict) else data
            if isinstance(payload, str):
                payload = json.loads(payload)
            msg = SemanticMsg.from_dict(payload)
        except (TypeError, ValueError) as exc:
            return ProcessResult.failed(str(exc))

        await self._message_work(msg).cancel_queued()
        await self._cleanup_local_artifact(msg)
        return ProcessResult.cancelled()

    async def _release_cancelled_semantic_lock(self, msg: SemanticMsg) -> None:
        if msg.lock_handoff is not None:
            lock_guard = PathLockGuard(get_viking_fs()._async_agfs)
            try:
                await lock_guard.adopt(msg.lock_handoff)
            except Exception as exc:
                logger.warning("Failed to release cancelled semantic lock: %s", exc)
            finally:
                await lock_guard.close()

    async def _resolve_skill_semantic_lock(
        self,
        msg: SemanticMsg,
        ctx: RequestContext,
        caller_lock: Optional[Dict[str, Any]],
    ) -> SemanticLockScope:
        scope = None

        async def acquire():
            nonlocal scope
            viking_fs = get_viking_fs()
            scope = await SemanticLockScope.resolve(
                msg.lock_handoff,
                caller_lock=caller_lock,
                ctx=ctx,
                fallback_path_factory=lambda: viking_fs._uri_to_path(msg.uri, ctx=ctx),
            )
            if scope.lock is None and await viking_fs.exists(msg.uri, ctx=ctx):
                lease = await viking_fs._async_agfs.pathlock_acquire_tree(
                    viking_fs._uri_to_path(msg.uri, ctx=ctx)
                )
                scope = SemanticLockScope(lease, _owned=True)
            return scope

        try:
            return await run_to_completion(acquire)
        except asyncio.CancelledError:
            # Acquiring/adopting a lease can itself outlive a cancelled await.
            if scope is not None:
                await run_to_completion(scope.close)
            raise

    def get_tree_stats(self) -> Optional["SemanticTreeStats"]:
        return SemanticTreeExecutor.get_active_stats()

    async def _process_memory_directory(
        self,
        msg: SemanticMsg,
        ctx: Optional[RequestContext] = None,
        lock: Optional[Dict[str, Any]] = None,
    ) -> None:
        """Process a memory directory with special handling.

        For memory directories:
        - Generate file summaries in bounded batches
        - Enqueue each changed file for vectorization as soon as its summary is ready
        - Retain only compact summaries for the directory overview
        - Vectorize the generated abstract.md and overview.md

        Args:
            msg: The semantic message containing directory info and changes
        """
        viking_fs = get_viking_fs()
        dir_uri = msg.uri
        ctx = ctx or self._default_ctx
        llm_sem = asyncio.Semaphore(self.max_concurrent_llm)

        try:
            entries = await viking_fs.ls(dir_uri, node_limit=LS_ALL_NODES, ctx=ctx)
        except Exception as e:
            raise RuntimeError(f"Failed to list memory directory {dir_uri}: {e}") from e

        file_paths: List[str] = []
        for entry in entries:
            name = entry.get("name", "")
            if not name or name.startswith(".") or name in [".", ".."]:
                continue
            if not entry.get("isDir", False):
                item_uri = VikingURI(dir_uri).join(name).uri
                file_paths.append(item_uri)
        file_paths.sort()

        if not file_paths:
            logger.info(f"No memory files found in {dir_uri}")
            return

        existing_summaries: Dict[str, str] = {}
        if msg.changes:
            try:
                old_overview = await viking_fs.read_file(f"{dir_uri}/.overview.md", ctx=ctx)
                if old_overview:
                    existing_summaries = self._parse_overview_md(body_for_preview(old_overview))
                    logger.info(
                        f"Parsed {len(existing_summaries)} existing summaries from overview.md"
                    )
            except Exception as e:
                logger.debug(f"No existing overview.md found for {dir_uri}: {e}")

        changed_files: Set[str] = set()
        if msg.changes:
            changed_files = set(msg.changes.get("added", []) + msg.changes.get("modified", []))
            deleted_files = set(msg.changes.get("deleted", []))
            logger.info(
                f"Processing memory directory {dir_uri} with changes: "
                f"added={len(msg.changes.get('added', []))}, "
                f"modified={len(msg.changes.get('modified', []))}, "
                f"deleted={len(deleted_files)}"
            )

        pending_indices: List[Tuple[int, str]] = []
        file_summaries: List[Optional[Dict[str, str]]] = [None] * len(file_paths)
        paths_to_vectorize = changed_files if msg.changes else set(file_paths)

        for idx, file_path in enumerate(file_paths):
            file_name = file_path.split("/")[-1]
            if file_path not in changed_files and file_name in existing_summaries:
                file_summaries[idx] = {
                    "name": file_name,
                    "summary": existing_summaries[file_name],
                }
                logger.debug(f"Reused existing summary for {file_name}")
            else:
                pending_indices.append((idx, file_path))

        if file_paths and not pending_indices:
            try:
                from openviking.metrics.datasources.cache import CacheEventDataSource

                CacheEventDataSource.record_hit("L1")
            except Exception:
                pass
        elif file_paths and pending_indices:
            try:
                from openviking.metrics.datasources.cache import CacheEventDataSource

                if len(file_paths) > len(pending_indices):
                    CacheEventDataSource.record_hit("L1")
                CacheEventDataSource.record_miss("L1")
            except Exception:
                pass

        if pending_indices:
            logger.info(
                f"Generating summaries for {len(pending_indices)} changed files "
                f"(reused {len(file_paths) - len(pending_indices)} cached)"
            )

            async def _gen(idx: int, file_path: str) -> None:
                file_name = file_path.split("/")[-1]
                try:
                    summary_dict = await self._generate_single_file_summary(
                        file_path, llm_sem=llm_sem, ctx=ctx
                    )
                    logger.debug(f"Generated summary for {file_name}")
                except Exception as e:
                    logger.warning(f"Failed to generate summary for {file_path}: {e}")
                    summary_dict = {"name": file_name, "summary": ""}

                if file_path in paths_to_vectorize and not msg.skip_vectorization:
                    await self._vectorize_single_file(
                        parent_uri=dir_uri,
                        context_type="memory",
                        file_path=file_path,
                        summary_dict=summary_dict,
                        ctx=ctx,
                        preserve_existing_created_at=True,
                        telemetry_id=msg.telemetry_id,
                    )
                file_summaries[idx] = {
                    "name": str(summary_dict.get("name") or file_name),
                    "summary": str(summary_dict.get("summary") or ""),
                }

            batch_size = max(1, min(self.max_concurrent_llm, 10))
            for batch_start in range(0, len(pending_indices), batch_size):
                batch = pending_indices[batch_start : batch_start + batch_size]
                logger.info(
                    f"[MemorySemantic] Processing batch {batch_start // batch_size + 1}/"
                    f"{(len(pending_indices) + batch_size - 1) // batch_size} "
                    f"({len(batch)} files)"
                )
                await asyncio.gather(*[_gen(i, fp) for i, fp in batch])

        completed_summaries = [s for s in file_summaries if s is not None]
        generated_content = await self._generate_overview(
            dir_uri,
            completed_summaries,
            [],
            llm_sem=llm_sem,
            total_files=len(file_paths),
            ctx=ctx,
        )
        overview, abstract = self._normalize_overview_generation(generated_content)

        try:
            wrote_semantics = await self._write_memory_directory_semantics(
                msg=msg,
                viking_fs=viking_fs,
                dir_uri=dir_uri,
                overview=overview,
                abstract=abstract,
                ctx=ctx,
                lock=lock,
                total_entries=len(file_paths),
                sampled_entries=len(completed_summaries),
            )
        except LockAcquisitionError:
            raise
        except Exception as e:
            raise RuntimeError(f"Failed to write abstract/overview for {dir_uri}: {e}") from e
        if not wrote_semantics.wrote:
            return
        logger.info(f"Generated abstract.md and overview.md for {dir_uri}")

        if msg.skip_vectorization:
            logger.info(f"Skipping vectorization for {dir_uri} (requested via SemanticMsg)")
            return
        if not (wrote_semantics.overview_body_changed or wrote_semantics.abstract_body_changed):
            logger.info(
                "Skipping directory vectorization for %s (visible semantics unchanged)",
                dir_uri,
            )
            return
        await self._vectorize_directory(
            uri=dir_uri,
            context_type="memory",
            abstract=abstract,
            overview=overview,
            ctx=ctx,
            telemetry_id=msg.telemetry_id,
        )
        logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")

    async def _write_memory_directory_semantics(
        self,
        *,
        msg: SemanticMsg,
        viking_fs: Any,
        dir_uri: str,
        overview: str,
        abstract: str,
        ctx: Optional[RequestContext],
        lock: Optional[Dict[str, Any]] = None,
        total_entries: int = 0,
        sampled_entries: int = 0,
    ) -> AbstractOverviewWriteResult:
        return await write_abstract_overview(
            viking_fs=viking_fs,
            dir_uri=dir_uri,
            overview=overview,
            abstract=abstract,
            ctx=ctx,
            is_stale=lambda: is_semantic_msg_stale(msg),
            metadata={
                **({"source": msg.source} if msg.source else {}),
                "generated_by": {
                    "component": "SemanticProcessor",
                    "trigger": msg.generation_trigger,
                },
                "freshness": freshness_metadata(total_entries, sampled_entries),
            },
            lock=lock,
            log_prefix="[MemorySemantic]",
        )

    async def _sync_topdown_recursive(
        self,
        root_uri: str,
        target_uri: str,
        ctx: Optional[RequestContext] = None,
        file_change_status: Optional[Dict[str, bool]] = None,
        lock: Optional[Dict[str, Any]] = None,
    ) -> SyncDiff:
        """Merge a temp/staging source tree into the target via VikingFS.sync_tree.

        The pure filesystem diff-and-move is delegated to
        :py:meth:`VikingFS.sync_tree`; this wrapper carries over parser
        sidecars (``.image_mappings.json``) and rewrites markdown image
        references once the visible files have been moved into place.
        Hidden entries are skipped by ``sync_tree``, which is why the
        sidecar handling lives here rather than in the FS layer.
        """
        viking_fs = get_viking_fs()
        if not await viking_fs.exists(root_uri, ctx=ctx):
            raise FileNotFoundError(
                f"Semantic source no longer exists; refusing to sync into {target_uri}: {root_uri}"
            )

        target_exists = await viking_fs.exists(target_uri, ctx=ctx)
        diff = await viking_fs.sync_tree(
            root_uri,
            target_uri,
            ctx=ctx,
            file_change_status=file_change_status,
            lease_ref=lock,
            delete_temp_after=False,
        )

        if not target_exists:
            # The whole temp tree (including the hidden .image_mappings.json
            # sidecar) was moved into the target; rewrite local image paths now.
            await self._rewrite_target_image_uris(root_uri, target_uri, ctx=ctx, lock=lock)
            return diff

        # sync_tree skips hidden files, so the .image_mappings.json sidecar is
        # still at the temp root. Carry it over and rewrite the synced markdown
        # before the temp tree is deleted below.
        await self._rewrite_target_image_uris(root_uri, target_uri, ctx=ctx, lock=lock)
        try:
            await viking_fs.delete_temp(root_uri, ctx=ctx)
        except Exception as e:
            logger.error(f"[SyncDiff] Failed to delete root directory {root_uri}: {e}")
        return diff

    async def _rewrite_target_image_uris(
        self,
        root_uri: str,
        target_uri: str,
        ctx: Optional[RequestContext] = None,
        lock: Optional[Dict[str, Any]] = None,
    ) -> None:
        """Rewrite local image refs in the target after a temp-to-target sync.

        ``_sync_topdown_recursive`` MOVES the visible files into the target and
        skips hidden ones, so afterwards the temp tree holds only the
        ``.image_mappings.json`` sidecars written by the parser (one per
        document root, possibly nested for directory ingests). Discovery is
        therefore driven by the markdown files already synced into the TARGET:
        their ancestor directories, mirrored back onto the temp tree, are where
        sidecars can live. Carry each one over (when missing) so
        :func:`rewrite_image_uris` can resolve local image paths against the
        images that were synced into the final target.
        """
        viking_fs = get_viking_fs()
        root_prefix = root_uri.rstrip("/")
        target_prefix = target_uri.rstrip("/")
        mapping_name = IMAGE_MAPPINGS_FILENAME

        if root_prefix != target_prefix:
            try:
                glob_result = await viking_fs.glob("**/*.md", uri=target_prefix, ctx=ctx)
                target_md_uris = glob_result.get("matches", [])
            except Exception:
                target_md_uris = []

            # Ancestor dirs of the target md files, as paths relative to the
            # target root — the candidate sidecar locations on both trees.
            candidate_rels = set()
            for md_uri in target_md_uris:
                d = md_uri.rsplit("/", 1)[0]
                while d == target_prefix or d.startswith(target_prefix + "/"):
                    candidate_rels.add(d[len(target_prefix) :].lstrip("/"))
                    if d == target_prefix:
                        break
                    d = d.rsplit("/", 1)[0]

            for rel in candidate_rels:
                src_mapping = (
                    f"{root_prefix}/{rel}/{mapping_name}"
                    if rel
                    else f"{root_prefix}/{mapping_name}"
                )
                target_mapping = (
                    f"{target_prefix}/{rel}/{mapping_name}"
                    if rel
                    else f"{target_prefix}/{mapping_name}"
                )
                try:
                    await viking_fs.stat(target_mapping, ctx=ctx, skip_count=True)
                    continue  # already carried over
                except Exception:
                    pass
                try:
                    mapping_content = await viking_fs.read_file(src_mapping, ctx=ctx)
                except Exception:
                    continue  # no sidecar at this level
                try:
                    # TODO: This must be optimized once pathlock is pushed down into ragfs.
                    await viking_fs.write_file(
                        target_mapping,
                        mapping_content,
                        ctx=ctx,
                        lease_ref=lock,
                    )
                except Exception:
                    # Target subtree may not exist (doc removed in sync); skip.
                    pass

        try:
            await rewrite_image_uris(target_uri, ctx=ctx, lease_ref=lock)
        except Exception as e:
            logger.error(f"[SyncDiff] Failed to rewrite image URIs for {target_uri}: {e}")

    async def _generate_text_summary(
        self,
        file_path: str,
        file_name: str,
        llm_sem: asyncio.Semaphore,
        ctx: Optional[RequestContext] = None,
        file_content: Optional[bytes] = None,
        materialize_content: bool = False,
    ) -> Dict[str, Any]:
        """Generate summary for a single text file (code, documentation, or other text)."""
        viking_fs = get_viking_fs()
        active_ctx = ctx or self._default_ctx

        content = (
            file_content
            if file_content is not None
            else await viking_fs.read_file(file_path, ctx=active_ctx)
        )
        if isinstance(content, bytes):
            from openviking.utils.embedding_utils import _decode_text_bytes

            content = _decode_text_bytes(content)

        def result(summary: str) -> Dict[str, Any]:
            return {"name": file_name, "summary": summary}

        config = get_openviking_config()

        # Limit content length
        max_chars = config.semantic.max_file_content_chars
        if len(content) > max_chars:
            content = content[:max_chars] + "\n...(truncated)"

        # Detect file type and select appropriate prompt
        file_type = self._detect_file_type(file_name)

        if file_type == FILE_TYPE_CODE:
            from openviking.parse.parsers.code.ast import extract_skeleton_result

            extraction = extract_skeleton_result(file_name, content)
            if extraction.text:
                skeleton_text = extraction.text
                max_skeleton_chars = config.semantic.max_skeleton_chars
                if len(skeleton_text) > max_skeleton_chars:
                    skeleton_text = skeleton_text[:max_skeleton_chars]
                return result(skeleton_text)
            prompt_id = "semantic.code_summary"
        elif file_type == FILE_TYPE_DOCUMENTATION:
            prompt_id = "semantic.document_summary"
        else:
            prompt_id = "semantic.file_summary"

        vlm = await self._get_vlm_config(active_ctx)
        if not vlm.is_available():
            if file_type == FILE_TYPE_CODE:
                logger.warning("VLM not available for code summary fallback: %s", file_path)
            else:
                logger.warning("VLM not available, using empty summary")
            return result("")

        from openviking.session.memory.utils.language import resolve_output_language

        output_language = resolve_output_language(content, config=config)
        prompt = render_prompt(
            prompt_id,
            {"file_name": file_name, "content": content, "output_language": output_language},
        )

        async with llm_sem:
            with bind_telemetry_stage("semantic_execute"):
                summary = await vlm.get_completion_async(prompt)
        return result(summary.strip())

    async def _generate_single_file_summary(
        self,
        file_path: str,
        llm_sem: Optional[asyncio.Semaphore] = None,
        ctx: Optional[RequestContext] = None,
        file_content: Optional[bytes] = None,
    ) -> Dict[str, Any]:
        """Generate summary for a single file.

        Args:
            file_path: File path

        Returns:
            {"name": file_name, "summary": summary_content}
        """
        file_name = file_path.split("/")[-1]
        llm_sem = llm_sem or asyncio.Semaphore(self.max_concurrent_llm)
        media_type = get_media_type(file_name, None)
        if file_name.lower().endswith(".ts"):
            try:
                prefix = (
                    file_content[:MPEG_TS_PROBE_BYTES]
                    if file_content is not None
                    else await get_viking_fs().read(
                        file_path,
                        offset=0,
                        size=MPEG_TS_PROBE_BYTES,
                        ctx=ctx,
                    )
                )
            except Exception:
                prefix = None
            media_type = get_media_type(file_name, None, content=prefix)
        if media_type in {"image", "audio", "video"}:
            vlm = await self._get_vlm_config(ctx or self._default_ctx)
            if media_type == "image":
                return await generate_image_summary(
                    file_path,
                    file_name,
                    llm_sem,
                    ctx=ctx,
                    vlm=vlm,
                )
            if media_type == "audio":
                return await generate_audio_summary(
                    file_path,
                    file_name,
                    llm_sem,
                    ctx=ctx,
                    vlm=vlm,
                )
            return await generate_video_summary(
                file_path,
                file_name,
                llm_sem,
                ctx=ctx,
                vlm=vlm,
            )
        return await self._generate_text_summary(
            file_path, file_name, llm_sem, ctx=ctx, file_content=file_content
        )

    def _child_summary_line(
        self,
        dir_uri: str,
        idx: int,
        item: Dict[str, str],
        link_map: Dict[str, str],
    ) -> str:
        """Render a subdirectory summary line with a collision-free link placeholder."""
        placeholder = f"viking://input_sample_c{idx}"
        link_map[placeholder] = self._markdown_link_target(dir_uri, item["name"])
        return f"- {item['name']}/ (link: {placeholder}): {item['abstract']}"

    @staticmethod
    def _markdown_link_target(dir_uri: str, entry_name: str) -> str:
        """Build a Markdown-safe target without changing the stored Viking URI."""
        entry_uri = VikingURI(dir_uri).join(entry_name).uri
        return markdown_safe_viking_uri(entry_uri)

    def _replace_link_references(self, generated_content: str, link_map: Dict[str, str]) -> str:
        """Resolve link placeholders (viking://input_sample_fN / cN) to real URIs.

        The model is fed compact, collision-free placeholders and asked to emit
        Markdown links against them; here we map each placeholder back to the
        entry's real URI. Unknown placeholders are left untouched.
        """

        def replace_link(match):
            return link_map.get(match.group(0), match.group(0))

        return re.sub(r"viking://input_sample_[fc]\d+", replace_link, generated_content)

    def _truncate_generated_text(self, text: str, max_chars: int) -> str:
        if max_chars <= 0 or len(text) <= max_chars:
            return text

        if max_chars <= 3:
            return text[:max_chars]

        first_sentence_end = None
        last_sentence_end_within_limit = None
        for sentence_end_match in re.finditer(r"\.(?!\d)(?=\s|$)|[!?](?=\s|$)|[。?!]", text):
            sentence_end = sentence_end_match.end()
            if first_sentence_end is None:
                first_sentence_end = sentence_end
            if sentence_end <= max_chars:
                last_sentence_end_within_limit = sentence_end
            elif last_sentence_end_within_limit is not None:
                break

        if last_sentence_end_within_limit is not None:
            return text[:last_sentence_end_within_limit].strip()
        if first_sentence_end is not None:
            return text[:first_sentence_end].strip()

        candidate = text[: max_chars - 3].rstrip()
        word_boundary = candidate.rfind(" ")
        if word_boundary > 0:
            return candidate[:word_boundary].rstrip() + "..."

        return candidate + "..."

    def _extract_abstract_from_overview(self, overview_content: str) -> str:
        """Extract an abstract from the Markdown overview brief description."""
        overview_content = body_for_preview(overview_content)
        lines = overview_content.split("\n")

        # Skip header lines (starting with #)
        content_lines = []
        in_header = True

        for line in lines:
            if line.strip() == "---":
                continue
            if in_header and line.startswith("#"):
                continue
            elif in_header and line.strip():
                in_header = False

            if not in_header:
                # Stop at first ##
                if line.startswith("##"):
                    break
                if line.strip():
                    content_lines.append(line.strip())

        return "\n".join(content_lines).strip()

    def _normalize_overview_generation(self, generated_content: str) -> Tuple[str, str]:
        """Convert raw Markdown overview output into final L1 overview and L0 abstract."""
        overview = body_for_preview(generated_content)
        abstract = self._extract_abstract_from_overview(overview)
        return self._enforce_size_limits(overview, abstract)

    def _enforce_size_limits(self, overview: str, abstract: str) -> Tuple[str, str]:
        """Enforce max size limits on overview and abstract."""
        semantic = get_openviking_config().semantic
        if len(overview) > semantic.overview_max_chars:
            overview = self._truncate_generated_text(overview, semantic.overview_max_chars)
        if len(abstract) > semantic.abstract_max_chars:
            abstract = self._truncate_generated_text(abstract, semantic.abstract_max_chars)
        return overview, abstract

    @classmethod
    def _parse_overview_md(cls, overview_content: str) -> Dict[str, str]:
        """Parse overview.md and extract direct-file summaries."""
        return parse_overview_file_summaries(overview_content)

    async def _generate_overview(
        self,
        dir_uri: str,
        file_summaries: List[Dict[str, str]],
        children_abstracts: List[Dict[str, str]],
        llm_sem: Optional[asyncio.Semaphore] = None,
        total_files: Optional[int] = None,
        total_children: Optional[int] = None,
        ctx: Optional[RequestContext] = None,
    ) -> str:
        """Generate raw directory overview model output.

        For small directories, generates a single overview from all file summaries.
        For large directories that would exceed the prompt budget, splits file
        summaries into batches, generates a partial overview per batch, then
        merges the partials into a final overview.

        Args:
            dir_uri: Directory URI
            file_summaries: File summary list (only entries participating in
                this aggregation, i.e. already sampled when applicable)
            children_abstracts: Subdirectory summary list (sampled when applicable)
            total_files: Total direct files in the directory (may exceed
                ``len(file_summaries)`` when the directory was sampled)
            total_children: Total direct subdirectories (may exceed
                ``len(children_abstracts)`` when sampled)

        Returns:
            Markdown overview content generated by the model.
        """

        config = get_openviking_config()
        active_ctx = ctx or self._default_ctx
        vlm = await self._get_vlm_config(active_ctx)
        semantic = config.semantic

        if not vlm.is_available():
            logger.warning("VLM not available, using default overview")
            return f"# {dir_uri.split('/')[-1]}\n\n[Directory overview is not ready]"

        from openviking.session.memory.utils.language import resolve_output_language

        # Build coverage statement so the model knows the directory scale
        # and whether the provided summaries represent all entries or a sample.
        total_files = len(file_summaries) if total_files is None else total_files
        total_children = len(children_abstracts) if total_children is None else total_children
        total_direct = total_files + total_children
        provided = len(file_summaries) + len(children_abstracts)
        if total_direct <= provided:
            directory_coverage = (
                f"Total direct entries: {total_direct}. "
                "All direct entries are represented in the summaries below."
            )
        else:
            directory_coverage = (
                f"Total direct entries: {total_direct} "
                f"({total_files} files, {total_children} subdirectories).\n"
                f"Summaries provided for this aggregation: {provided}.\n"
                f"Direct entries not individually shown: {total_direct - provided}.\n"
                "Coverage: sampled. The summaries below are representative "
                "entries; generalize cautiously and do not treat them as exhaustive."
            )

        # Build link placeholder mapping and summary string.
        # The model is fed collision-free placeholders (viking://input_sample_fN)
        # and asked to emit Markdown links; placeholders are resolved to real
        # URIs in post-processing. This avoids the fragility of bare [N] indices
        # and of asking the model to reproduce long/CJK URIs verbatim.
        link_map: Dict[str, str] = {}
        file_summaries_lines = []
        for idx, item in enumerate(file_summaries, 1):
            placeholder = f"viking://input_sample_f{idx}"
            link_map[placeholder] = self._markdown_link_target(dir_uri, item["name"])
            file_summaries_lines.append(
                f"- {item['name']} (link: {placeholder}): {item['summary']}"
            )
        file_summaries_str = "\n".join(file_summaries_lines) if file_summaries_lines else "None"

        # Build subdirectory summary string
        children_abstracts_str = (
            "\n".join(
                self._child_summary_line(dir_uri, idx, item, link_map)
                for idx, item in enumerate(children_abstracts, 1)
            )
            if children_abstracts
            else "None"
        )

        language_source_parts = []
        if file_summaries:
            language_source_parts.append(file_summaries_str)
        if children_abstracts:
            language_source_parts.append(children_abstracts_str)
        if not language_source_parts:
            language_source_parts.append(dir_uri.split("/")[-1])
        output_language = resolve_output_language("\n".join(language_source_parts), config=config)

        # Budget guard: check if prompt would be oversized
        estimated_size = len(file_summaries_str) + len(children_abstracts_str)
        over_budget = estimated_size > semantic.max_overview_prompt_chars
        entry_count = len(file_summaries) + len(children_abstracts)
        many_entries = entry_count > semantic.overview_batch_size

        if over_budget and many_entries:
            # Many entries, oversized prompt → batch and merge
            logger.info(
                f"Overview prompt for {dir_uri} exceeds budget "
                f"({estimated_size} chars, {len(file_summaries)} files, "
                f"{len(children_abstracts)} subdirectories). "
                f"Splitting into batches of {semantic.overview_batch_size}."
            )
            overview = await self._batched_generate_overview(
                dir_uri,
                file_summaries,
                children_abstracts,
                link_map,
                llm_sem=llm_sem,
                output_language=output_language,
                directory_coverage=directory_coverage,
                vlm=vlm,
            )
        elif over_budget:
            # Few files but long summaries → truncate summaries to fit budget
            logger.info(
                f"Overview prompt for {dir_uri} exceeds budget "
                f"({estimated_size} chars) with {len(file_summaries)} files. "
                f"Truncating summaries to fit."
            )
            budget = semantic.max_overview_prompt_chars
            budget -= len(children_abstracts_str)
            per_file = max(100, budget // max(len(file_summaries), 1))
            truncated_lines = []
            for idx, item in enumerate(file_summaries, 1):
                summary = item["summary"][:per_file]
                truncated_lines.append(
                    f"- {item['name']} (link: viking://input_sample_f{idx}): {summary}"
                )
            file_summaries_str = "\n".join(truncated_lines)
            overview = await self._single_generate_overview(
                dir_uri,
                file_summaries_str,
                children_abstracts_str,
                link_map,
                output_language=output_language,
                directory_coverage=directory_coverage,
                vlm=vlm,
            )
        else:
            overview = await self._single_generate_overview(
                dir_uri,
                file_summaries_str,
                children_abstracts_str,
                link_map,
                output_language=output_language,
                directory_coverage=directory_coverage,
                vlm=vlm,
            )

        return overview

    async def _single_generate_overview(
        self,
        dir_uri: str,
        file_summaries_str: str,
        children_abstracts_str: str,
        link_map: Dict[str, str],
        output_language: str = "en",
        directory_coverage: str = "",
        *,
        vlm: "VLMHandle",
    ) -> str:
        """Generate overview from a single prompt (small directories)."""

        try:
            prompt = render_prompt(
                "semantic.overview_generation",
                {
                    "dir_name": dir_uri.split("/")[-1],
                    "file_summaries": file_summaries_str,
                    "children_abstracts": children_abstracts_str,
                    "output_language": output_language,
                    "directory_coverage": directory_coverage,
                },
            )

            with bind_telemetry_stage("semantic_execute"):
                overview = await vlm.get_completion_async(prompt)

            overview = self._replace_link_references(overview, link_map)

            return overview.strip()

        except Exception as e:
            logger.error(
                f"Failed to generate overview for {dir_uri}: {e}",
                exc_info=True,
            )
            return f"# {dir_uri.split('/')[-1]}\n\n[Directory overview is not generated]"

    async def _batched_generate_overview(
        self,
        dir_uri: str,
        file_summaries: List[Dict[str, str]],
        children_abstracts: List[Dict[str, str]],
        link_map: Dict[str, str],
        llm_sem: Optional[asyncio.Semaphore] = None,
        output_language: str = "en",
        directory_coverage: str = "",
        *,
        vlm: "VLMHandle",
    ) -> str:
        """Generate overview by batching file and subdirectory summaries.

        Splits both input kinds into batches, generates a partial overview per
        batch, then merges the partials without repeating the raw inputs.
        """
        config = get_openviking_config()
        semantic = config.semantic
        batch_size = semantic.overview_batch_size
        dir_name = dir_uri.split("/")[-1]

        work_items = [("file", index, item) for index, item in enumerate(file_summaries, 1)] + [
            ("child", index, item) for index, item in enumerate(children_abstracts, 1)
        ]
        batches = [work_items[i : i + batch_size] for i in range(0, len(work_items), batch_size)]
        logger.info(f"Generating overview for {dir_uri} in {len(batches)} batches")

        # Generate partial overviews concurrently using global link placeholders.
        if llm_sem is None:
            llm_sem = asyncio.Semaphore(self.max_concurrent_llm)
        partial_overviews = [None] * len(batches)
        batch_prompts: List[Tuple[int, str, Dict[str, str]]] = []

        for batch_idx, batch in enumerate(batches):
            batch_lines = []
            child_lines = []
            batch_link_map: Dict[str, str] = {}
            for entry_kind, global_idx, item in batch:
                assert global_idx is not None
                if entry_kind == "file":
                    placeholder = f"viking://input_sample_f{global_idx}"
                    batch_link_map[placeholder] = link_map.get(
                        placeholder, self._markdown_link_target(dir_uri, item["name"])
                    )
                    batch_lines.append(f"- {item['name']} (link: {placeholder}): {item['summary']}")
                else:
                    child_lines.append(
                        self._child_summary_line(dir_uri, global_idx, item, batch_link_map)
                    )

            prompt = render_prompt(
                "semantic.overview_generation",
                {
                    "dir_name": dir_name,
                    "file_summaries": "\n".join(batch_lines) or "None",
                    "children_abstracts": "\n".join(child_lines) or "None",
                    "output_language": output_language,
                    "directory_coverage": directory_coverage,
                },
            )
            batch_prompts.append((batch_idx, prompt, batch_link_map))

        async def _run_batch(batch_idx: int, prompt: str, batch_link_map: Dict[str, str]) -> None:
            try:
                async with llm_sem:
                    with bind_telemetry_stage("semantic_execute"):
                        partial = await vlm.get_completion_async(prompt)
                partial = self._replace_link_references(partial, batch_link_map)
                partial_overviews[batch_idx] = partial.strip()
            except Exception as e:
                logger.warning(
                    f"Failed to generate partial overview batch "
                    f"{batch_idx + 1}/{len(batches)} for {dir_uri}: {e}"
                )

        await asyncio.gather(*[_run_batch(*bp) for bp in batch_prompts])
        partial_overviews = [p for p in partial_overviews if p is not None]

        if not partial_overviews:
            return f"# {dir_name}\n\n[Directory overview is not generated]"

        # If only one batch succeeded, use it directly
        if len(partial_overviews) == 1:
            return partial_overviews[0]

        # Merge partials only; each child abstract is already represented once.
        # Partials already contain real URIs, but the shared generation prompt can
        # make the merge model emit placeholders again. Resolve the merged output
        # with the complete map before persisting it.
        combined = "\n\n---\n\n".join(partial_overviews)
        try:
            prompt = render_prompt(
                "semantic.overview_generation",
                {
                    "dir_name": dir_name,
                    "file_summaries": combined,
                    "children_abstracts": "None",
                    "output_language": output_language,
                    "directory_coverage": directory_coverage,
                },
            )
            with bind_telemetry_stage("semantic_execute"):
                overview = await vlm.get_completion_async(prompt)
            overview = self._replace_link_references(overview, link_map)
            return overview.strip()
        except Exception as e:
            logger.error(
                f"Failed to merge partial overviews for {dir_uri}: {e}",
                exc_info=True,
            )
            return partial_overviews[0]

    async def _skill_root_semantics(
        self,
        uri: str,
        *,
        ctx: RequestContext,
        regenerate: bool = False,
        lock: Optional[Dict[str, Any]] = None,
        skill_content: str | bytes | None = None,
    ) -> Tuple[str, str]:
        """Keep the package root tied only to its SKILL.md definition."""
        viking_fs = get_viking_fs()
        if (
            not regenerate
            and await viking_fs.exists(f"{uri}/.abstract.md", ctx=ctx)
            and await viking_fs.exists(f"{uri}/.overview.md", ctx=ctx)
        ):
            abstract = body_for_preview(await viking_fs.read_file(f"{uri}/.abstract.md", ctx=ctx))
            overview = body_for_preview(await viking_fs.read_file(f"{uri}/.overview.md", ctx=ctx))
            if abstract and overview:
                return overview, abstract

        from openviking.core.skill_loader import SkillLoader
        from openviking.utils.skill_processor import SkillProcessor

        content = (
            skill_content
            if skill_content is not None
            else await viking_fs.read_file(f"{uri}/SKILL.md", ctx=ctx)
        )
        if isinstance(content, bytes):
            content = content.decode("utf-8")
        definition = SkillLoader.parse(content)
        processor = SkillProcessor(vikingdb=None, vlm_resolver=self._vlm_resolver)
        abstract = processor._build_skill_abstract(definition)
        overview = await processor._generate_overview(
            definition,
            await self._get_vlm_config(ctx),
        )
        await run_to_completion(
            lambda: write_abstract_overview(
                viking_fs=viking_fs,
                dir_uri=uri,
                abstract=abstract,
                overview=overview,
                ctx=ctx,
                lock=lock,
                is_stale=lambda: False,
                metadata={
                    "generated_by": {"component": "SkillProcessor", "trigger": "skill_refresh"}
                },
            )
        )
        return overview, abstract

    async def _vectorize_directory(
        self,
        uri: str,
        context_type: str,
        abstract: str,
        overview: str,
        ctx: Optional[RequestContext] = None,
        ingest_options: IngestOptions | None = None,
        skill_source_path: str = "",
        scalar_overrides: Optional[Dict[int, Dict[str, Any]]] = None,
        actions: Optional[Dict[int, str]] = None,
        field_patches: Optional[Dict[int, FieldPatch]] = None,
        include_abstract: bool = True,
        include_overview: bool = True,
        telemetry_id: str | None = None,
        md5s: Optional[Dict[int, str]] = None,
    ) -> set[int]:
        """Create directory Context and enqueue to EmbeddingQueue."""

        from openviking.utils.embedding_utils import vectorize_directory_meta

        active_ctx = ctx or self._default_ctx
        skill_meta = None
        if context_type == "skill" and classify_uri(uri).is_skill_root:
            import yaml

            parsed = yaml.safe_load(body_for_preview(abstract))
            if isinstance(parsed, dict):
                skill_meta = {
                    key: parsed.get(key, [] if key in {"tags", "allowed_tools"} else "")
                    for key in ("name", "description", "tags", "allowed_tools")
                }
                skill_meta["source_path"] = skill_source_path
        return await vectorize_directory_meta(
            uri=uri,
            abstract=abstract,
            overview=overview,
            context_type=context_type,
            ctx=active_ctx,
            ingest_options=ingest_options,
            content_is_body=context_type == "skill",
            **({"meta": skill_meta} if skill_meta is not None else {}),
            scalar_overrides=scalar_overrides,
            actions=actions,
            field_patches=field_patches,
            include_abstract=include_abstract,
            include_overview=include_overview,
            telemetry_id=telemetry_id,
            md5s=md5s,
        )

    async def _load_transfer_file_summaries(
        self,
        file_paths: List[str],
        ctx: Optional[RequestContext] = None,
    ) -> Dict[str, str]:
        """Load copied/moved file summaries from their existing target L2 vectors."""
        if not file_paths:
            return {}
        viking_fs = get_viking_fs()
        vector_store = viking_fs._get_vector_store()
        if vector_store is None:
            return {}
        active_ctx = ctx or self._default_ctx
        return await vector_store.get_l2_abstracts_by_uris(file_paths, ctx=active_ctx)

    async def _update_vector_fields(
        self,
        *,
        record_id: str,
        uri: str,
        level: int,
        field_patch: FieldPatch,
        ctx: RequestContext,
        telemetry_id: str | None = None,
    ) -> bool:
        from openviking.storage.queuefs import get_queue_manager
        from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
        from openviking.telemetry import get_current_telemetry
        from openviking.utils.embedding_utils import _enqueue_embedding_message

        embedding_msg = EmbeddingMsg.for_update_fields(
            record_id=record_id,
            field_patch=field_patch,
            context_data={
                "uri": uri,
                "level": level,
                "account_id": ctx.account_id,
                "owner_user_id": ctx.user.user_id,
            },
            telemetry_id=(
                get_current_telemetry().telemetry_id if telemetry_id is None else telemetry_id
            ),
        )
        queue_manager = get_queue_manager()
        embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING, allow_create=True)
        return await _enqueue_embedding_message(
            embedding_queue,
            embedding_msg,
        )

    async def _vectorize_single_file(
        self,
        parent_uri: str,
        context_type: str,
        file_path: str,
        summary_dict: Dict[str, str],
        ctx: Optional[RequestContext] = None,
        use_summary: bool = False,
        preserve_existing_created_at: bool = False,
        ingest_options: IngestOptions | None = None,
        file_md5: Optional[str] = None,
        file_content: Optional[bytes] = None,
        materialize_content: bool = False,
        scalar_override: Optional[Dict[str, Any]] = None,
        field_patch: FieldPatch | None = None,
        action: str = "merge",
        telemetry_id: str | None = None,
    ) -> bool:
        """Vectorize a single file using its content or summary."""
        from openviking.utils.embedding_utils import vectorize_file

        active_ctx = ctx or self._default_ctx
        return await vectorize_file(
            file_path=file_path,
            summary_dict=summary_dict,
            parent_uri=parent_uri,
            context_type=context_type,
            ctx=active_ctx,
            use_summary=use_summary,
            preserve_existing_created_at=preserve_existing_created_at,
            ingest_options=ingest_options,
            file_md5=file_md5,
            file_content=file_content,
            materialize_content=materialize_content,
            scalar_override=scalar_override,
            field_patch=field_patch,
            action=action,
            telemetry_id=telemetry_id,
        )