Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

10.3 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Tree Builder for OpenViking.

Converts parsed document trees into OpenViking context objects with proper
L0/L1/L2 content and URI structure.

v5.0 Architecture:
1. Parser: parse + create directory structure in temp VikingFS
2. TreeBuilder: build final URI metadata and keep temp references
3. ResourceProcessor: source commit from temp to final VikingFS path
4. SemanticProcessor: async generate L0/L1 + vectorize

IMPORTANT (v5.0 Architecture):
- Parser creates directory structure directly, no LLM calls
- TreeBuilder does not move files; source commit is handled after URI metadata is built
- SemanticProcessor handles all semantic generation asynchronously
- Temporary directory approach eliminates memory pressure and enables concurrency
- Resource objects are lightweight (no content fields)
- Content splitting is handled by Parser, not TreeBuilder
"""

from typing import Any, Optional

from openviking.core.building_tree import BuildingTree
from openviking.core.context import Context
from openviking.core.namespace import is_content_root_uri
from openviking.parse.parsers.media.utils import get_media_base_uri, get_media_type
from openviking.server.identity import RequestContext
from openviking.storage.viking_fs import get_viking_fs
from openviking.utils import parse_code_hosting_url
from openviking.utils.path_safety import normalize_storage_target_uri
from openviking_cli.utils import get_logger
from openviking_cli.utils.uri import VikingURI

logger = get_logger(__name__)


class TreeBuilder:
    """
    Builds OpenViking context tree from parsed documents (v5.0).

    New v5.0 Architecture:
    - Parser creates directory structure in temp VikingFS (no LLM calls)
    - TreeBuilder builds final URI metadata while preserving temp URIs
    - ResourceProcessor commits temp content to the final source path
    - SemanticProcessor handles semantic generation asynchronously

    Process flow:
    1. Parser creates directory structure with files in temp VikingFS
    2. TreeBuilder.finalize_from_temp() returns final URI and temp URI metadata
    3. ResourceProcessor performs source commit with short path locks
    4. SemanticProcessor generates .abstract.md and .overview.md asynchronously
    5. SemanticProcessor directly vectorizes and inserts to collection

    Key changes from v4.0:
    - Semantic generation moved from Parser to SemanticQueue
    - ResourceProcessor enqueues directories for async processing
    - Direct vectorization in SemanticProcessor (no EmbeddingQueue)
    """

    def __init__(self):
        """Initialize TreeBuilder."""
        pass

    def _get_base_uri(
        self, scope: str, source_path: Optional[str] = None, source_format: Optional[str] = None
    ) -> str:
        """Get base URI for scope, with special handling for media files."""
        # Check if it's a media file first
        if scope == "resources":
            media_type = get_media_type(source_path, source_format)
            if media_type:
                return get_media_base_uri(media_type)
            return "viking://resources"
        raise ValueError(f"unsupported tree scope: {scope}")

    # ============================================================================
    # v5.0 Methods (temporary directory + SemanticQueue architecture)
    # ============================================================================

    async def resolve_target_uri(
        self,
        *,
        ctx: RequestContext,
        doc_name: str,
        scope: str = "resources",
        to_uri: Optional[str] = None,
        parent_uri: Optional[str] = None,
        source_path: Optional[str] = None,
        source_format: Optional[str] = None,
        create_parent: bool = False,
    ) -> tuple[str, Optional[str]]:
        """Resolve the final target URI and optional unique-name candidate."""

        if to_uri:
            to_uri = normalize_storage_target_uri(to_uri)
        if parent_uri:
            parent_uri = normalize_storage_target_uri(parent_uri)

        final_doc_name = VikingURI.sanitize_segment(doc_name)
        if source_path and source_format == "repository":
            parsed_org_repo = parse_code_hosting_url(source_path)
            if parsed_org_repo:
                final_doc_name = parsed_org_repo

        auto_base_uri = self._get_base_uri(scope, source_path, source_format)
        base_uri = parent_uri or auto_base_uri
        use_to_as_parent = bool(to_uri and is_content_root_uri(to_uri, kind="resource"))
        if to_uri and not use_to_as_parent:
            return to_uri, None

        effective_parent_uri = (parent_uri or to_uri) if use_to_as_parent else parent_uri
        if effective_parent_uri:
            effective_parent_uri = effective_parent_uri.rstrip("/")
        if effective_parent_uri:
            viking_fs = get_viking_fs()
            parent_is_content_root = is_content_root_uri(
                effective_parent_uri,
                kind="resource",
            )
            parent_exists = await viking_fs.exists(effective_parent_uri, ctx=ctx)
            if not parent_exists:
                if create_parent or parent_is_content_root:
                    logger.info(
                        f"[TreeBuilder] Parent URI does not exist, creating: {effective_parent_uri}"
                    )
                    await viking_fs.mkdir(effective_parent_uri, exist_ok=True, ctx=ctx)
                else:
                    raise FileNotFoundError(
                        f"Parent URI does not exist: {effective_parent_uri}. "
                        f"Use --parent-auto-create/-p to automatically create it."
                    )
            stat_result = await viking_fs.stat(effective_parent_uri, ctx=ctx, skip_count=True)
            if not stat_result.get("isDir"):
                raise ValueError(f"Parent URI is not a directory: {effective_parent_uri}")
            base_uri = effective_parent_uri

        planned_uri = VikingURI(base_uri).join(final_doc_name).uri
        return planned_uri, planned_uri

    async def finalize_from_temp(
        self,
        temp_dir_path: str,
        ctx: RequestContext,
        scope: str = "resources",
        to_uri: Optional[str] = None,
        parent_uri: Optional[str] = None,
        source_path: Optional[str] = None,
        source_format: Optional[str] = None,
        create_parent: bool = False,
        flatten_single_file: bool = False,
        artifact_ref: Optional[Any] = None,
        output_store: Optional[Any] = None,
    ) -> "BuildingTree":
        """
        Finalize URI metadata for a temp parse result.

        Args:
            to_uri: Exact target URI, or resources root to import under
            parent_uri: Target parent URI (must exist unless create_parent is True)
            create_parent: Whether to automatically create parent directory if it doesn't exist
            artifact_ref / output_store: When provided, the artifact structure is
                read through the backend-agnostic store (supporting local
                artifacts); otherwise the legacy AGFS ``ls(temp_uri)`` path is
                used. Either way the resulting ``temp_doc_uri`` keeps the AGFS URI
                shape and target-URI resolution stays on VikingFS.
        """

        viking_fs = get_viking_fs()
        temp_uri = temp_dir_path

        # 1. Find the single document root. The structural walk is identical for
        # AGFS and local artifacts, so it is delegated to resolve_artifact_doc_root
        # when a store/ref is available. ``temp_doc_uri`` remains an AGFS-shaped
        # URI relative to temp_uri for downstream persistence.
        if output_store is not None and artifact_ref is not None:
            from openviking.parse.output import resolve_artifact_doc_root

            resolved = await resolve_artifact_doc_root(
                output_store, artifact_ref, flatten_single_file=flatten_single_file
            )
            original_name = resolved.doc_name
            doc_name = VikingURI.sanitize_segment(original_name)
            temp_doc_uri = f"{temp_uri.rstrip('/')}/{resolved.doc_rel}"
            root_is_file = resolved.root_is_file
        else:
            entries = await viking_fs.ls(temp_uri, ctx=ctx)
            doc_dirs = [e for e in entries if e.get("isDir") and e["name"] not in [".", ".."]]

            if len(doc_dirs) != 1:
                logger.error(
                    f"[TreeBuilder] Expected 1 document directory in {temp_uri}, found {len(doc_dirs)}"
                )
                raise ValueError(
                    f"[TreeBuilder] Expected 1 document directory in {temp_uri}, found {len(doc_dirs)}"
                )

            original_name = doc_dirs[0]["name"]
            doc_name = VikingURI.sanitize_segment(original_name)
            temp_doc_uri = f"{temp_uri}/{original_name}"  # use original name to find temp dir
            root_is_file = False
            if flatten_single_file:
                doc_entries = [
                    entry
                    for entry in await viking_fs.ls(temp_doc_uri, ctx=ctx)
                    if entry.get("name") not in [".", ".."]
                ]
                if len(doc_entries) == 1 and not doc_entries[0].get("isDir"):
                    original_name = doc_entries[0]["name"]
                    doc_name = VikingURI.sanitize_segment(original_name)
                    temp_doc_uri = f"{temp_doc_uri}/{original_name}"
                    root_is_file = True
        if original_name != doc_name:
            logger.debug(f"[TreeBuilder] Sanitized doc name: {original_name!r} -> {doc_name!r}")

        planned_uri, unique_candidate_uri = await self.resolve_target_uri(
            ctx=ctx,
            doc_name=original_name,
            scope=scope,
            to_uri=to_uri,
            parent_uri=parent_uri,
            source_path=source_path,
            source_format=source_format,
            create_parent=create_parent,
        )

        tree = BuildingTree(
            source_path=source_path,
            source_format=source_format,
        )
        tree._root_uri = planned_uri
        tree._root_is_file = root_is_file
        if unique_candidate_uri:
            tree._candidate_uri = unique_candidate_uri

        # Create a minimal Context object for the root so that tree.root is not None
        root_context = Context(uri=planned_uri, temp_uri=temp_doc_uri)
        tree.add_context(root_context)

        return tree