Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

20.8 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Unified resource processor with strategy-based routing."""

import time
from collections.abc import Awaitable
from pathlib import Path
from typing import TYPE_CHECKING, Optional
from urllib.parse import urlparse

from openviking.metrics.datasources.resource import ResourceIngestionEventDataSource
from openviking.parse.accessors.base import LocalResource, SourceType
from openviking.parse.accessors.mime_types import IANA_MEDIA_TYPE_TO_EXTENSION
from openviking.parse.backend import ParserBackend, normalize_parser_backend
from openviking.parse.base import ParseResult
from openviking.parse.mode import ParseMode, normalize_parse_mode
from openviking.parse.parsers.constants import (
    CODE_EXTENSIONS,
    DOCUMENTATION_EXTENSIONS,
    IGNORE_EXTENSIONS,
    MPEG_TS_EXTENSION_ALIAS,
    TYPESCRIPT_MPEG_TS_EXTENSION,
)
from openviking.parse.parsers.media.utils import is_mpeg_ts, read_mpeg_ts_probe
from openviking.parse.parsers.upload_utils import is_empty_file
from openviking.parse.registry import parse
from openviking.server.local_input_guard import (
    is_remote_resource_source,
    looks_like_local_path,
)
from openviking.telemetry import get_current_telemetry
from openviking_cli.exceptions import InvalidArgumentError, PermissionDeniedError
from openviking_cli.utils.logger import get_logger

# All known valid extensions - only these should be stripped when getting stem
# Build from multiple sources:
# 1. IANA media type mappings (comprehensive)
# 2. Code/documentation/ignore extensions (parser-specific)
KNOWN_EXTENSIONS: set[str] = set()
for extensions in IANA_MEDIA_TYPE_TO_EXTENSION.values():
    KNOWN_EXTENSIONS.update(extensions)
KNOWN_EXTENSIONS.update(CODE_EXTENSIONS)
KNOWN_EXTENSIONS.update(DOCUMENTATION_EXTENSIONS)
KNOWN_EXTENSIONS.update(IGNORE_EXTENSIONS)


def _smart_stem(path_or_name: str | Path) -> str:
    """Get the stem of a filename, but only strip known valid extensions.

    For filenames like "2601.00014" where ".00014" is not a valid extension,
    returns the full name instead of just "2601".

    Args:
        path_or_name: Path object or string filename

    Returns:
        Stem with only known extensions stripped
    """
    path = Path(path_or_name)
    suffix = path.suffix.lower()

    if suffix in KNOWN_EXTENSIONS:
        return path.stem

    # If the suffix is not a known extension, treat the whole name as the stem
    return path.name


if TYPE_CHECKING:
    from openviking.parse.vlm import VLMProcessor
    from openviking_cli.utils.storage import StoragePath

logger = get_logger(__name__)


class UnifiedResourceProcessor:
    """Unified resource processing for files, URLs, and raw content.

    Uses two-layer architecture:
    - Phase 1: AccessorRegistry gets LocalResource from source
    - Phase 2: ParserRegistry parses LocalResource to ParseResult
    """

    def __init__(
        self,
        vlm_processor: Optional["VLMProcessor"] = None,
        storage: Optional["StoragePath"] = None,
    ):
        self.storage = storage
        self._vlm_processor = vlm_processor
        self._accessor_registry = None

    def _get_vlm_processor(self) -> Optional["VLMProcessor"]:
        if self._vlm_processor is None:
            raise RuntimeError(
                "UnifiedResourceProcessor requires an explicitly configured VLMProcessor"
            )
        return self._vlm_processor

    def _get_accessor_registry(self):
        """Lazy initialize AccessorRegistry for two-layer mode."""
        if self._accessor_registry is None:
            from openviking.parse.accessors import get_accessor_registry

            self._accessor_registry = get_accessor_registry()
        return self._accessor_registry

    def _get_parser_router(self):
        """Get ParserRouter."""
        if not hasattr(self, "_parser_router"):
            from openviking.parse.parser_router import ParserRouter
            from openviking.parse.registry import get_registry

            self._parser_router = ParserRouter(get_registry())
        return self._parser_router

    def understanding_api_enabled(self) -> bool:
        return self._get_parser_router().understanding_api_enabled()

    def should_use_understanding_api(self, source: str | Path | LocalResource) -> bool:
        if isinstance(source, LocalResource):
            if source.path.is_dir():
                return False
            return self._get_parser_router().should_use_understanding_api(
                source,
                resolved_extension=str(source.meta.get("resolved_extension") or ""),
            )
        return self._get_parser_router().should_use_understanding_api(source)

    def should_use_understanding_directly(self, source: str, **kwargs) -> bool:
        return self._get_parser_router().should_use_understanding_directly(source, **kwargs)

    def durable_route_requires_preparation(
        self,
        source: str,
        *,
        parse_mode: ParseMode | str = ParseMode.DEFAULT,
        **kwargs,
    ) -> bool:
        """Return whether durable parser selection must inspect fetched content."""
        if normalize_parse_mode(parse_mode) is ParseMode.NO_SPLIT:
            return False
        parser_backend = normalize_parser_backend(kwargs.get("parser_backend"))
        if parser_backend is ParserBackend.INTERNAL:
            return False

        from openviking.parse.accessors.feishu_accessor import FeishuAccessor
        from openviking.parse.accessors.web_feed_accessor import WebFeedAccessor

        accessor = self._get_accessor_registry().get_accessor(source, **kwargs)
        if isinstance(accessor, (FeishuAccessor, WebFeedAccessor)):
            return False

        router = self._get_parser_router()
        if not router.understanding_api_enabled():
            return False
        if parser_backend is ParserBackend.UNDERSTANDING:
            return True
        if not Path(urlparse(source).path).suffix:
            return True
        return router.should_use_understanding_api(source)

    async def submit_understanding(self, source: str | Path | LocalResource, **kwargs) -> str:
        return await self._get_parser_router().submit(source, **kwargs)

    async def upload_understanding_file(self, source: str | Path | LocalResource) -> str:
        return await self._get_parser_router().upload_file(source)

    @staticmethod
    def _set_resolved_identity(resource: LocalResource, source_name: Optional[str]) -> None:
        meta = resource.meta
        if resource.path.is_dir():
            extension = ""
        elif resource.source_type == SourceType.HTTP:
            extension = str(meta.get("extension") or resource.path.suffix)
        else:
            extension = Path(source_name).suffix if source_name else ""
            extension = extension or str(meta.get("extension") or resource.path.suffix)

        extension = extension.lower()
        if (
            extension == TYPESCRIPT_MPEG_TS_EXTENSION
            and resource.path.is_file()
            and is_mpeg_ts(read_mpeg_ts_probe(resource.path))
        ):
            extension = MPEG_TS_EXTENSION_ALIAS

        meta["resolved_extension"] = extension
        meta["resolved_name"] = source_name or meta.get("original_filename") or resource.path.name

    async def prepare(
        self,
        source: str,
        allow_local_path_resolution: bool = True,
        **kwargs,
    ) -> LocalResource:
        """Fetch a path/URL and freeze the identity used for parser routing."""
        if (
            not allow_local_path_resolution
            and not is_remote_resource_source(source)
            and looks_like_local_path(source)
        ):
            raise PermissionDeniedError(
                "HTTP server only accepts remote resource URLs or temp-uploaded files; "
                "direct host filesystem paths are not allowed."
            )

        from openviking.parse.accessors.feishu_accessor import FeishuAccessor
        from openviking.parse.feishu_import import recursive_wiki

        if FeishuAccessor._is_feishu_url(str(source)) and (
            FeishuAccessor._parse_feishu_url(str(source))[0] == "folder"
            or (
                FeishuAccessor._parse_feishu_url(str(source))[0] == "wiki"
                and recursive_wiki(kwargs)
            )
        ):
            backend = normalize_parser_backend(kwargs.get("parser_backend"))
            mode = normalize_parse_mode(kwargs.get("parse_mode", ParseMode.DEFAULT))
            kwargs["_feishu_use_understanding"] = bool(
                mode is ParseMode.DEFAULT
                and backend is not ParserBackend.INTERNAL
                and (
                    backend is ParserBackend.UNDERSTANDING
                    or self._get_parser_router().should_use_understanding_api(source)
                )
            )
        resource = await self._get_accessor_registry().access(source, **kwargs)
        try:
            self._set_resolved_identity(resource, kwargs.get("source_name"))
            self._reject_empty_resource(resource, kwargs.get("source_name"))
        except Exception:
            # Ownership transfers to the caller only after prepare succeeds.
            # Until then, release temporary accessor results on every error.
            resource.cleanup()
            raise
        return resource

    @staticmethod
    def _reject_empty_resource(resource: LocalResource, source_name: Optional[str]) -> None:
        """Refuse a source with no content, wherever it was fetched from.

        Every ingestion path reaches this method: prepare_durable_source
        freezes a source here before the request touches the tree, and
        process() calls it for anything that was not frozen earlier. An empty
        file is refused once, at the point the bytes are first in hand, rather
        than being parsed and indexed as an empty resource.

        Directories are skipped: their size is not meaningful, and
        directory_scan already drops empty or whitespace-only members. content/write is a
        different path and still allows an empty file, because creating one
        there is an explicit user action.
        """
        try:
            if not resource.path.is_file() or not is_empty_file(resource.path):
                return
        except OSError:
            # An unreadable source is not this check's business; let the normal
            # ingestion path report it.
            return

        meta = resource.meta
        name = (
            source_name
            or meta.get("resolved_name")
            or meta.get("original_filename")
            or resource.path.name
        )
        raise InvalidArgumentError(
            f"'{name}' is empty or contains only whitespace, so there is nothing to extract or index. "
            "Add a resource with content, or use content/write if an empty file is "
            "what you want."
        )

    async def process(
        self,
        source: str,
        instruction: str = "",
        allow_local_path_resolution: bool = True,
        prepared_resource: Optional[LocalResource] = None,
        parse_mode: ParseMode | str = ParseMode.DEFAULT,
        **kwargs,
    ) -> ParseResult:
        """Process any source (file/URL/content) with two-layer architecture.

        Phase 1: Use AccessorRegistry to get LocalResource
        Phase 2: Use ParserRegistry to parse LocalResource

        Resource Lifecycle:
        - Temporary resources are managed via context manager or temp_dir_path
        - Directories needed for TreeBuilder are preserved via ParseResult.temp_dir_path
        """

        metrics_account_id = kwargs.pop("_metrics_account_id", None)

        async def run_stage(stage: str, operation: Awaitable[ParseResult | LocalResource]):
            started_at = time.perf_counter()
            status = "ok"
            try:
                with get_current_telemetry().measure(f"resource.{stage}"):
                    return await operation
            except Exception:
                status = "error"
                raise
            finally:
                ResourceIngestionEventDataSource.record_stage(
                    stage=stage,
                    status=status,
                    duration_seconds=time.perf_counter() - started_at,
                    account_id=metrics_account_id,
                )

        mode = normalize_parse_mode(parse_mode)

        # First check if source is raw content (not URL/path)
        is_potential_path = (
            allow_local_path_resolution and len(source) <= 1024 and "\n" not in source
        )
        if not is_potential_path and not self._is_url(source):
            # Treat as raw content
            return await run_stage(
                "parse_artifact",
                parse(
                    source,
                    instruction=instruction,
                    split_content=mode is ParseMode.DEFAULT,
                ),
            )

        if (
            mode is ParseMode.DEFAULT
            and prepared_resource is None
            and self._has_prepared_understanding_response(kwargs)
        ):
            parse_kwargs = dict(kwargs)
            parse_kwargs.pop("tos_signature", None)
            parse_kwargs.pop("tos_access", None)
            parse_kwargs["instruction"] = instruction
            parse_kwargs["vlm_processor"] = self._get_vlm_processor()
            parse_kwargs["storage"] = self.storage
            parse_kwargs["_source_meta"] = {
                "source_type": SourceType.HTTP,
                "original_url": source,
            }
            parse_kwargs["original_source"] = source
            parse_kwargs["parser_backend"] = ParserBackend.UNDERSTANDING
            parse_kwargs.pop("feishu_access_token", None)

            explicit_name = kwargs.get("resource_name") or kwargs.get("source_name")
            if explicit_name:
                parse_kwargs["resource_name"] = _smart_stem(explicit_name)
            return await run_stage(
                "parse_artifact", self._get_parser_router().parse(source, **parse_kwargs)
            )

        if (
            mode is ParseMode.DEFAULT
            and prepared_resource is None
            and self.should_use_understanding_directly(source, **kwargs)
        ):
            parse_kwargs = dict(kwargs)
            parse_kwargs.pop("tos_signature", None)
            parse_kwargs.pop("tos_access", None)
            parse_kwargs["instruction"] = instruction
            parse_kwargs["vlm_processor"] = self._get_vlm_processor()
            parse_kwargs["storage"] = self.storage
            parse_kwargs["_source_meta"] = {
                "source_type": SourceType.HTTP,
                "original_url": source,
            }
            parse_kwargs["original_source"] = source

            explicit_name = kwargs.get("resource_name") or kwargs.get("source_name")
            if explicit_name:
                parse_kwargs["resource_name"] = _smart_stem(explicit_name)
            return await run_stage(
                "parse_artifact", self._get_parser_router().parse(source, **parse_kwargs)
            )

        # Phase 1: Accessor - get local resource. A caller may prepare a remote
        # resource first so async routing uses the same detected file type.
        if prepared_resource is not None:
            local_resource = prepared_resource
        else:
            local_resource = await run_stage(
                "source_prepare",
                self.prepare(
                    source,
                    allow_local_path_resolution=allow_local_path_resolution,
                    **({"parse_mode": mode} if mode is ParseMode.NO_SPLIT else {}),
                    **kwargs,
                ),
            )

        # Use context manager for automatic cleanup, but preserve directories for TreeBuilder
        try:
            # Phase 2: Parser - parse the local resource
            parse_kwargs = dict(kwargs)
            # Source credentials are consumed by the accessor. Never forward
            # them into parser kwargs, parse results, or later queue payloads.
            parse_kwargs.pop("auth_config", None)
            parse_kwargs.pop("github_token", None)
            parse_kwargs.pop("tos_signature", None)
            parse_kwargs.pop("tos_access", None)
            parse_kwargs["instruction"] = instruction
            parse_kwargs["_source_meta"] = local_resource.meta
            parse_kwargs["resolved_extension"] = kwargs.get(
                "resolved_extension"
            ) or local_resource.meta.get("resolved_extension", "")
            # CRITICAL: Pass along the original_source!
            # This is the full URL the user provided (e.g. "https://github.com/volcengine/OpenViking")
            # CodeRepositoryParser and TreeBuilder need this to extract the org/repo format
            # Without it, we'd only get the repo name without the org prefix!
            parse_kwargs["original_source"] = local_resource.original_source

            # Set resource_name from source_name or path
            source_name = kwargs.get("source_name")
            if source_name:
                parse_kwargs["resource_name"] = _smart_stem(source_name)
                parse_kwargs.setdefault("source_name", source_name)
            else:
                # For git repositories, use repo_name from meta if available
                repo_name = local_resource.meta.get("repo_name")
                if repo_name and local_resource.source_type == SourceType.GIT:
                    # Use the last part of repo_name as the resource_name (e.g., "OpenViking" from "volcengine/OpenViking")
                    parse_kwargs["resource_name"] = repo_name.split("/")[-1]
                else:
                    # Prefer original_filename from meta for HTTP downloads
                    original_filename = local_resource.meta.get("original_filename")
                    if original_filename:
                        parse_kwargs.setdefault("resource_name", _smart_stem(original_filename))
                        parse_kwargs.setdefault("source_name", original_filename)
                    else:
                        parse_kwargs.setdefault("resource_name", _smart_stem(local_resource.path))

            if mode is ParseMode.NO_SPLIT:
                parse_kwargs["split_content"] = False

            parse_kwargs["vlm_processor"] = self._get_vlm_processor()
            parse_kwargs["storage"] = self.storage

            # If it's a directory, use DirectoryParser which will delegate to CodeRepositoryParser if it's a git repo
            if local_resource.path.is_dir():
                from openviking.parse.parsers.directory import DirectoryParser

                parser = DirectoryParser()
                parse_kwargs["_feishu_import_plan"] = local_resource.feishu_plan

                result = await run_stage(
                    "parse_artifact", parser.parse(str(local_resource.path), **parse_kwargs)
                )
                # Preserve temporary directory for TreeBuilder
                if local_resource.is_temporary and not result.temp_dir_path:
                    result.temp_dir_path = str(local_resource.path)
                    # Mark as non-temporary so context manager doesn't clean it up
                    local_resource.is_temporary = False
                return result

            # For files, use ParserRouter to decide which parser to use
            parser_router = self._get_parser_router()
            if (
                mode is ParseMode.NO_SPLIT
                and local_resource.source_type == SourceType.FEISHU
                and local_resource.meta.get("feishu_content_kind") == "file"
            ):
                parse_kwargs["parser_backend"] = ParserBackend.INTERNAL
            parse_kwargs.pop("feishu_access_token", None)
            parse_kwargs.pop("lark_file", None)
            checkpoint = parse_kwargs.pop("_feishu_checkpoint", None)
            if checkpoint is not None and local_resource.meta.get("feishu_content_kind") == "file":
                saved, save = checkpoint
                if "file" in saved:
                    parse_kwargs["understanding_response_id"] = saved["file"]

                async def record(response_id):
                    await save("file", response_id)

                parse_kwargs["_response_checkpoint"] = record
            return await run_stage(
                "parse_artifact", parser_router.parse(local_resource, **parse_kwargs)
            )
        finally:
            # Clean up temporary resources unless they need to be preserved
            local_resource.cleanup()

    def _is_url(self, source: str) -> bool:
        """Check if source is a URL."""
        return is_remote_resource_source(source)

    @staticmethod
    def _has_prepared_understanding_response(kwargs: dict) -> bool:
        from openviking.parse.understanding_api import (
            PREPARED_FILE_ID_ARG,
            PREPARED_RESPONSE_ID_ARG,
        )

        return any(
            isinstance(kwargs.get(key), str) and bool(kwargs[key].strip())
            for key in (PREPARED_RESPONSE_ID_ARG, PREPARED_FILE_ID_ARG)
        )