Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

16.4 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Summarizer for OpenViking.

Handles summarization and key information extraction.
"""

from collections import defaultdict
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple

from openviking.core.namespace import context_type_for_uri
from openviking.storage.pathlock_transfer import PathLockGuard
from openviking.storage.queuefs import SemanticMsg, get_queue_manager
from openviking.storage.queuefs.semantic_msg import build_semantic_coalesce_key
from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.telemetry import get_current_telemetry
from openviking.utils.ingest_options import IngestOptions
from openviking_cli.utils import get_logger
from openviking_cli.utils.uri import VikingURI

if TYPE_CHECKING:
    from openviking.parse.vlm import VLMProcessor
    from openviking.server.identity import RequestContext

logger = get_logger(__name__)


@dataclass(frozen=True)
class FileRefreshRequest:
    """One committed flat-file change awaiting parent semantic maintenance.

    This is an execution-time input for the shared summarizer, not a durable
    RNFV or queue schema.  A batch coordinator collects one request per
    successfully committed file and lets :meth:`refresh_file_parents` coalesce
    them by direct parent.
    """

    file_uri: str
    created: bool = False
    md5: str | None = None
    abstract: str = ""
    vector_action: str = ""
    ingest_options: IngestOptions = field(default_factory=IngestOptions)

    def __post_init__(self) -> None:
        if self.vector_action not in {"", "upsert", "merge"}:
            raise ValueError(f"unsupported file vector action: {self.vector_action}")


class Summarizer:
    """
    Handles summarization of resources.
    """

    def __init__(self, vlm_processor: "VLMProcessor"):
        self.vlm_processor = vlm_processor

    async def refresh_file_parent(
        self,
        *,
        file_uri: str,
        ctx: "RequestContext",
        lock_handoff: Dict[str, Any] | None = None,
        skip_vectorization: bool = False,
        ingest_options: IngestOptions | None = None,
        created: bool = False,
        file_md5: str | None = None,
        file_abstract: str = "",
        file_vector_action: str = "",
        generation_trigger: str = "semantic_refresh",
        force_refresh: bool | None = None,
    ) -> Dict[str, Any]:
        """Summarize one flat file and refresh its parent directory semantics.

        ``force_refresh`` controls parent-directory aggregation. ``None`` (the
        default used by resource ingest) always aggregates immediately. A boolean
        opts into the freshness gate: the changed file is still summarized and
        vectorized, but parent L0/L1 aggregation is deferred by the freshness
        policy unless ``force_refresh`` is ``True``. The returned
        ``semantic_action`` reports the gate decision so callers can surface a
        ``deferred``/``skipped`` status.
        """
        parent = VikingURI(file_uri).parent
        if parent is None:
            return {"status": "error", "message": f"file has no parent URI: {file_uri}"}
        result = await self.refresh_file_parents(
            changes=(
                FileRefreshRequest(
                    file_uri=file_uri,
                    created=created,
                    md5=file_md5,
                    abstract=file_abstract,
                    vector_action=file_vector_action,
                    ingest_options=IngestOptions.from_value(ingest_options),
                ),
            ),
            ctx=ctx,
            lock_handoff=lock_handoff,
            skip_vectorization=skip_vectorization,
            ingest_options=ingest_options,
            generation_trigger=generation_trigger,
            force_refresh=force_refresh,
        )
        return {
            "status": "success",
            "enqueued_count": result["enqueued_count"],
            "semantic_action": result["parent_actions"][parent.uri.rstrip("/")],
        }

    async def refresh_file_parents(
        self,
        *,
        changes: Sequence[FileRefreshRequest],
        ctx: "RequestContext",
        lock_handoff: Dict[str, Any] | None = None,
        skip_vectorization: bool = False,
        ingest_options: IngestOptions | None = None,
        generation_trigger: str = "semantic_refresh",
        force_refresh: bool | None = None,
    ) -> Dict[str, Any]:
        """Queue flat-file semantic maintenance grouped by each direct parent.

        A file remains the RNFV planning unit, while this execution helper owns
        the batch-only concern: one freshness decision and at most one
        ``SemanticMsg`` for every direct parent directory.
        """
        from openviking.storage.abstract_overview import plan_abstract_overview_refresh
        from openviking.storage.errors import LockAcquisitionError
        from openviking.storage.queuefs.semantic_ops.freshness_policy import FreshnessAction
        from openviking_cli.utils.config import get_openviking_config

        grouped: dict[str, list[FileRefreshRequest]] = defaultdict(list)
        for change in changes:
            parent = VikingURI(change.file_uri).parent
            if parent is None:
                raise ValueError(f"file has no parent URI: {change.file_uri}")
            grouped[parent.uri.rstrip("/")].append(change)

        if not grouped:
            return {"status": "success", "enqueued_count": 0, "parent_actions": {}}
        if lock_handoff is not None and len(grouped) != 1:
            raise ValueError("one semantic lock handoff requires exactly one parent refresh")

        queue_manager = get_queue_manager()
        semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
        telemetry_id = get_current_telemetry().telemetry_id
        parent_actions: dict[str, str] = {}
        enqueued_count = 0
        for parent_uri, parent_changes in sorted(grouped.items()):
            context_types = {context_type_for_uri(change.file_uri) for change in parent_changes}
            if len(context_types) != 1:
                raise ValueError(f"mixed context types under parent refresh: {parent_uri}")
            context_type = context_types.pop()
            added = sorted(change.file_uri for change in parent_changes if change.created)
            modified = sorted(change.file_uri for change in parent_changes if not change.created)

            semantic_action = FreshnessAction.REFRESH_NOW
            if force_refresh is not None:
                semantic_config = get_openviking_config().semantic
                try:
                    decision = await plan_abstract_overview_refresh(
                        viking_fs=get_viking_fs(),
                        dir_uri=parent_uri,
                        changed_entries=len(parent_changes),
                        ctx=ctx,
                        overview_sample_limit=getattr(semantic_config, "overview_sample_limit", 32),
                        refresh_ratio=getattr(semantic_config, "freshness_refresh_ratio", 0.10),
                        force_refresh=force_refresh,
                    )
                    semantic_action = decision.action
                except LockAcquisitionError:
                    # Parent aggregation is best-effort; changed-file work still runs.
                    logger.info("Skipping busy parent semantic refresh: %s", parent_uri)
                    semantic_action = FreshnessAction.NOOP
            parent_actions[parent_uri] = semantic_action.value
            aggregate_directory = semantic_action is FreshnessAction.REFRESH_NOW

            msg = SemanticMsg(
                uri=parent_uri,
                context_type=context_type,
                recursive=False,
                account_id=ctx.account_id,
                user_id=ctx.user.user_id,
                peer_id=ctx.user.user_id,
                role=str(ctx.role),
                lock_handoff=lock_handoff,
                skip_vectorization=skip_vectorization,
                telemetry_id=telemetry_id,
                changes={
                    change_type: uris
                    for change_type, uris in (("added", added), ("modified", modified))
                    if uris
                },
                coalesce_key=(
                    build_semantic_coalesce_key(
                        context_type=context_type,
                        uri=parent_uri,
                        account_id=ctx.account_id,
                        user_id=ctx.user.user_id,
                        peer_id=ctx.user.user_id,
                    )
                    if aggregate_directory and lock_handoff is None
                    else ""
                ),
                generation_trigger=generation_trigger,
                aggregate_directory=aggregate_directory,
                ingest_options=ingest_options,
                file_md5s={change.file_uri: change.md5 for change in parent_changes if change.md5}
                or None,
                file_abstracts={
                    change.file_uri: change.abstract for change in parent_changes if change.abstract
                }
                or None,
                file_vector_actions={
                    change.file_uri: change.vector_action
                    for change in parent_changes
                    if change.vector_action
                },
                file_ingest_options={
                    change.file_uri: change.ingest_options
                    for change in parent_changes
                    if change.ingest_options != IngestOptions()
                },
            )
            enqueue_id = await semantic_queue.enqueue(msg)
            if enqueue_id == "deduplicated":
                continue
            else:
                enqueued_count += 1
        return {
            "status": "success",
            "enqueued_count": enqueued_count,
            "parent_actions": parent_actions,
        }

    async def summarize(
        self,
        resource_uris: List[str],
        ctx: "RequestContext",
        skip_vectorization: bool = False,
        lock: Optional[Dict[str, Any] | PathLockGuard] = None,
        **kwargs,
    ) -> Dict[str, Any]:
        """
        Summarize the given resources.
        Triggers SemanticQueue to generate .abstract.md and .overview.md.
        """
        queue_manager = get_queue_manager()
        semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)

        temp_uris = kwargs.get("temp_uris", [])
        ingest_options = IngestOptions.from_value(kwargs.get("ingest_options"))
        source = kwargs.get("semantic_source")
        generation_trigger = str(kwargs.get("generation_trigger") or "manual_refresh")
        # Pre-computed change set (local incremental import): when present, the
        # semantic tree restricts re-summarization/vectorization to these files
        # instead of diffing the whole tree.
        changes = kwargs.get("changes")
        # Per-file md5 (target-URI keyed) for those changed files, so the tree executor's
        # re-vectorization records the fresh fingerprint.
        file_md5s = kwargs.get("file_md5s") or {}
        artifact_ref = kwargs.get("artifact_ref")
        artifact_files = kwargs.get("artifact_files") or []
        file_abstracts = kwargs.get("file_abstracts") or {}
        semantic_plan = kwargs.get("semantic_plan")
        if not temp_uris:
            temp_uris = resource_uris
        if len(temp_uris) != len(resource_uris):
            logger.error(
                f"temp_uris length ({len(temp_uris)}) must match resource_uris length ({len(resource_uris)})"
            )
            return {
                "status": "error",
                "message": "temp_uris length must match resource_uris length",
            }
        if lock is not None and len(resource_uris) != 1:
            raise ValueError("one semantic lock handoff requires exactly one resource")
        enqueued_count = 0

        telemetry = get_current_telemetry()
        agfs = get_viking_fs()._async_agfs
        lock_guard = (
            lock
            if isinstance(lock, PathLockGuard)
            else PathLockGuard(agfs, lock) if lock is not None else None
        )
        lock_handoff: Optional[Dict[str, Any]] = None
        if lock_guard is not None:
            lock_handoff = await lock_guard.to_handoff()

        def is_resources_root(uri: str) -> bool:
            return (uri or "").rstrip("/") == "viking://resources"

        async def list_top_children(temp_uri: str) -> List[Tuple[str, str]]:
            viking_fs = get_viking_fs()
            entries = await viking_fs.ls(
                temp_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx
            )
            children: List[Tuple[str, str]] = []
            for entry in entries:
                name = entry.get("name", "")
                if not name or name in {".", ".."}:
                    continue
                child_temp_uri = VikingURI(temp_uri).join(name).uri
                children.append((name, child_temp_uri))
            return children

        for uri, temp_uri in zip(resource_uris, temp_uris, strict=True):
            # Determine context_type based on URI
            context_type = context_type_for_uri(uri)

            enqueue_units: List[Tuple[str, str]] = []
            if is_resources_root(uri) and uri != temp_uri and lock_handoff is None:
                children = await list_top_children(temp_uri)
                if not children:
                    return {
                        "status": "error",
                        "message": f"no top-level import items found under temp uri: {temp_uri}",
                    }
                for name, child_temp_uri in children:
                    child_target_uri = VikingURI("viking://resources").join(name).uri
                    enqueue_units.append((child_target_uri, child_temp_uri))
            else:
                enqueue_units.append((uri, temp_uri))

            for target_uri, source_uri in enqueue_units:
                msg = SemanticMsg(
                    uri=source_uri,
                    context_type=context_type,
                    account_id=ctx.account_id,
                    user_id=ctx.user.user_id,
                    group_ids=ctx.group_ids,
                    peer_id=ctx.user.user_id,
                    role=str(ctx.role),
                    skip_vectorization=skip_vectorization,
                    telemetry_id=telemetry.telemetry_id,
                    target_uri=target_uri if target_uri != source_uri else None,
                    lock_handoff=lock_handoff,
                    is_code_repo=kwargs.get("is_code_repo", False),
                    ingest_options=ingest_options,
                    source=source,
                    generation_trigger=generation_trigger,
                    changes=changes,
                    file_md5s=file_md5s,
                    artifact_ref=artifact_ref,
                    artifact_files=artifact_files,
                    file_abstracts=file_abstracts,
                    plan=semantic_plan,
                )
                try:
                    if lock_guard is None:
                        enqueue_id = await semantic_queue.enqueue(msg)
                    else:
                        enqueue_id = await lock_guard.handoff_and_dispatch(
                            lock_handoff,
                            lambda msg=msg: semantic_queue.enqueue(msg),
                            accepted=lambda result: result != "deduplicated",
                        )
                except BaseException:
                    if (
                        isinstance(lock, dict)
                        and lock_guard is not None
                        and lock_guard.lease is not None
                        and lock_guard.lease is not lock
                    ):
                        lock.clear()
                        lock.update(lock_guard.lease)
                    raise
                if enqueue_id == "deduplicated":
                    logger.info("Semantic generation already queued for: %s", target_uri)
                    continue
                enqueued_count += 1
                logger.info(
                    f"Enqueued semantic generation for: {target_uri} (skip_vectorization={skip_vectorization})"
                )

        return {
            "status": "success",
            "enqueued_count": enqueued_count,
            "lock_handed_off": lock_guard is not None and lock_guard.transferred,
        }