Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Unified resource processor with strategy-based routing."""
import time
from collections.abc import Awaitable
from pathlib import Path
from typing import TYPE_CHECKING, Optional
from urllib.parse import urlparse
from openviking.metrics.datasources.resource import ResourceIngestionEventDataSource
from openviking.parse.accessors.base import LocalResource, SourceType
from openviking.parse.accessors.mime_types import IANA_MEDIA_TYPE_TO_EXTENSION
from openviking.parse.backend import ParserBackend, normalize_parser_backend
from openviking.parse.base import ParseResult
from openviking.parse.mode import ParseMode, normalize_parse_mode
from openviking.parse.parsers.constants import (
CODE_EXTENSIONS,
DOCUMENTATION_EXTENSIONS,
IGNORE_EXTENSIONS,
MPEG_TS_EXTENSION_ALIAS,
TYPESCRIPT_MPEG_TS_EXTENSION,
)
from openviking.parse.parsers.media.utils import is_mpeg_ts, read_mpeg_ts_probe
from openviking.parse.parsers.upload_utils import is_empty_file
from openviking.parse.registry import parse
from openviking.server.local_input_guard import (
is_remote_resource_source,
looks_like_local_path,
)
from openviking.telemetry import get_current_telemetry
from openviking_cli.exceptions import InvalidArgumentError, PermissionDeniedError
from openviking_cli.utils.logger import get_logger
# All known valid extensions - only these should be stripped when getting stem
# Build from multiple sources:
# 1. IANA media type mappings (comprehensive)
# 2. Code/documentation/ignore extensions (parser-specific)
KNOWN_EXTENSIONS: set[str] = set()
for extensions in IANA_MEDIA_TYPE_TO_EXTENSION.values():
KNOWN_EXTENSIONS.update(extensions)
KNOWN_EXTENSIONS.update(CODE_EXTENSIONS)
KNOWN_EXTENSIONS.update(DOCUMENTATION_EXTENSIONS)
KNOWN_EXTENSIONS.update(IGNORE_EXTENSIONS)
def _smart_stem(path_or_name: str | Path) -> str:
"""Get the stem of a filename, but only strip known valid extensions.
For filenames like "2601.00014" where ".00014" is not a valid extension,
returns the full name instead of just "2601".
Args:
path_or_name: Path object or string filename
Returns:
Stem with only known extensions stripped
"""
path = Path(path_or_name)
suffix = path.suffix.lower()
if suffix in KNOWN_EXTENSIONS:
return path.stem
# If the suffix is not a known extension, treat the whole name as the stem
return path.name
if TYPE_CHECKING:
from openviking.parse.vlm import VLMProcessor
from openviking_cli.utils.storage import StoragePath
logger = get_logger(__name__)
class UnifiedResourceProcessor:
"""Unified resource processing for files, URLs, and raw content.
Uses two-layer architecture:
- Phase 1: AccessorRegistry gets LocalResource from source
- Phase 2: ParserRegistry parses LocalResource to ParseResult
"""
def __init__(
self,
vlm_processor: Optional["VLMProcessor"] = None,
storage: Optional["StoragePath"] = None,
):
self.storage = storage
self._vlm_processor = vlm_processor
self._accessor_registry = None
def _get_vlm_processor(self) -> Optional["VLMProcessor"]:
if self._vlm_processor is None:
raise RuntimeError(
"UnifiedResourceProcessor requires an explicitly configured VLMProcessor"
)
return self._vlm_processor
def _get_accessor_registry(self):
"""Lazy initialize AccessorRegistry for two-layer mode."""
if self._accessor_registry is None:
from openviking.parse.accessors import get_accessor_registry
self._accessor_registry = get_accessor_registry()
return self._accessor_registry
def _get_parser_router(self):
"""Get ParserRouter."""
if not hasattr(self, "_parser_router"):
from openviking.parse.parser_router import ParserRouter
from openviking.parse.registry import get_registry
self._parser_router = ParserRouter(get_registry())
return self._parser_router
def understanding_api_enabled(self) -> bool:
return self._get_parser_router().understanding_api_enabled()
def should_use_understanding_api(self, source: str | Path | LocalResource) -> bool:
if isinstance(source, LocalResource):
if source.path.is_dir():
return False
return self._get_parser_router().should_use_understanding_api(
source,
resolved_extension=str(source.meta.get("resolved_extension") or ""),
)
return self._get_parser_router().should_use_understanding_api(source)
def should_use_understanding_directly(self, source: str, **kwargs) -> bool:
return self._get_parser_router().should_use_understanding_directly(source, **kwargs)
def durable_route_requires_preparation(
self,
source: str,
*,
parse_mode: ParseMode | str = ParseMode.DEFAULT,
**kwargs,
) -> bool:
"""Return whether durable parser selection must inspect fetched content."""
if normalize_parse_mode(parse_mode) is ParseMode.NO_SPLIT:
return False
parser_backend = normalize_parser_backend(kwargs.get("parser_backend"))
if parser_backend is ParserBackend.INTERNAL:
return False
from openviking.parse.accessors.feishu_accessor import FeishuAccessor
from openviking.parse.accessors.web_feed_accessor import WebFeedAccessor
accessor = self._get_accessor_registry().get_accessor(source, **kwargs)
if isinstance(accessor, (FeishuAccessor, WebFeedAccessor)):
return False
router = self._get_parser_router()
if not router.understanding_api_enabled():
return False
if parser_backend is ParserBackend.UNDERSTANDING:
return True
if not Path(urlparse(source).path).suffix:
return True
return router.should_use_understanding_api(source)
async def submit_understanding(self, source: str | Path | LocalResource, **kwargs) -> str:
return await self._get_parser_router().submit(source, **kwargs)
async def upload_understanding_file(self, source: str | Path | LocalResource) -> str:
return await self._get_parser_router().upload_file(source)
@staticmethod
def _set_resolved_identity(resource: LocalResource, source_name: Optional[str]) -> None:
meta = resource.meta
if resource.path.is_dir():
extension = ""
elif resource.source_type == SourceType.HTTP:
extension = str(meta.get("extension") or resource.path.suffix)
else:
extension = Path(source_name).suffix if source_name else ""
extension = extension or str(meta.get("extension") or resource.path.suffix)
extension = extension.lower()
if (
extension == TYPESCRIPT_MPEG_TS_EXTENSION
and resource.path.is_file()
and is_mpeg_ts(read_mpeg_ts_probe(resource.path))
):
extension = MPEG_TS_EXTENSION_ALIAS
meta["resolved_extension"] = extension
meta["resolved_name"] = source_name or meta.get("original_filename") or resource.path.name
async def prepare(
self,
source: str,
allow_local_path_resolution: bool = True,
**kwargs,
) -> LocalResource:
"""Fetch a path/URL and freeze the identity used for parser routing."""
if (
not allow_local_path_resolution
and not is_remote_resource_source(source)
and looks_like_local_path(source)
):
raise PermissionDeniedError(
"HTTP server only accepts remote resource URLs or temp-uploaded files; "
"direct host filesystem paths are not allowed."
)
from openviking.parse.accessors.feishu_accessor import FeishuAccessor
from openviking.parse.feishu_import import recursive_wiki
if FeishuAccessor._is_feishu_url(str(source)) and (
FeishuAccessor._parse_feishu_url(str(source))[0] == "folder"
or (
FeishuAccessor._parse_feishu_url(str(source))[0] == "wiki"
and recursive_wiki(kwargs)
)
):
backend = normalize_parser_backend(kwargs.get("parser_backend"))
mode = normalize_parse_mode(kwargs.get("parse_mode", ParseMode.DEFAULT))
kwargs["_feishu_use_understanding"] = bool(
mode is ParseMode.DEFAULT
and backend is not ParserBackend.INTERNAL
and (
backend is ParserBackend.UNDERSTANDING
or self._get_parser_router().should_use_understanding_api(source)
)
)
resource = await self._get_accessor_registry().access(source, **kwargs)
try:
self._set_resolved_identity(resource, kwargs.get("source_name"))
self._reject_empty_resource(resource, kwargs.get("source_name"))
except Exception:
# Ownership transfers to the caller only after prepare succeeds.
# Until then, release temporary accessor results on every error.
resource.cleanup()
raise
return resource
@staticmethod
def _reject_empty_resource(resource: LocalResource, source_name: Optional[str]) -> None:
"""Refuse a source with no content, wherever it was fetched from.
Every ingestion path reaches this method: prepare_durable_source
freezes a source here before the request touches the tree, and
process() calls it for anything that was not frozen earlier. An empty
file is refused once, at the point the bytes are first in hand, rather
than being parsed and indexed as an empty resource.
Directories are skipped: their size is not meaningful, and
directory_scan already drops empty or whitespace-only members. content/write is a
different path and still allows an empty file, because creating one
there is an explicit user action.
"""
try:
if not resource.path.is_file() or not is_empty_file(resource.path):
return
except OSError:
# An unreadable source is not this check's business; let the normal
# ingestion path report it.
return
meta = resource.meta
name = (
source_name
or meta.get("resolved_name")
or meta.get("original_filename")
or resource.path.name
)
raise InvalidArgumentError(
f"'{name}' is empty or contains only whitespace, so there is nothing to extract or index. "
"Add a resource with content, or use content/write if an empty file is "
"what you want."
)
async def process(
self,
source: str,
instruction: str = "",
allow_local_path_resolution: bool = True,
prepared_resource: Optional[LocalResource] = None,
parse_mode: ParseMode | str = ParseMode.DEFAULT,
**kwargs,
) -> ParseResult:
"""Process any source (file/URL/content) with two-layer architecture.
Phase 1: Use AccessorRegistry to get LocalResource
Phase 2: Use ParserRegistry to parse LocalResource
Resource Lifecycle:
- Temporary resources are managed via context manager or temp_dir_path
- Directories needed for TreeBuilder are preserved via ParseResult.temp_dir_path
"""
metrics_account_id = kwargs.pop("_metrics_account_id", None)
async def run_stage(stage: str, operation: Awaitable[ParseResult | LocalResource]):
started_at = time.perf_counter()
status = "ok"
try:
with get_current_telemetry().measure(f"resource.{stage}"):
return await operation
except Exception:
status = "error"
raise
finally:
ResourceIngestionEventDataSource.record_stage(
stage=stage,
status=status,
duration_seconds=time.perf_counter() - started_at,
account_id=metrics_account_id,
)
mode = normalize_parse_mode(parse_mode)
# First check if source is raw content (not URL/path)
is_potential_path = (
allow_local_path_resolution and len(source) <= 1024 and "\n" not in source
)
if not is_potential_path and not self._is_url(source):
# Treat as raw content
return await run_stage(
"parse_artifact",
parse(
source,
instruction=instruction,
split_content=mode is ParseMode.DEFAULT,
),
)
if (
mode is ParseMode.DEFAULT
and prepared_resource is None
and self._has_prepared_understanding_response(kwargs)
):
parse_kwargs = dict(kwargs)
parse_kwargs.pop("tos_signature", None)
parse_kwargs.pop("tos_access", None)
parse_kwargs["instruction"] = instruction
parse_kwargs["vlm_processor"] = self._get_vlm_processor()
parse_kwargs["storage"] = self.storage
parse_kwargs["_source_meta"] = {
"source_type": SourceType.HTTP,
"original_url": source,
}
parse_kwargs["original_source"] = source
parse_kwargs["parser_backend"] = ParserBackend.UNDERSTANDING
parse_kwargs.pop("feishu_access_token", None)
explicit_name = kwargs.get("resource_name") or kwargs.get("source_name")
if explicit_name:
parse_kwargs["resource_name"] = _smart_stem(explicit_name)
return await run_stage(
"parse_artifact", self._get_parser_router().parse(source, **parse_kwargs)
)
if (
mode is ParseMode.DEFAULT
and prepared_resource is None
and self.should_use_understanding_directly(source, **kwargs)
):
parse_kwargs = dict(kwargs)
parse_kwargs.pop("tos_signature", None)
parse_kwargs.pop("tos_access", None)
parse_kwargs["instruction"] = instruction
parse_kwargs["vlm_processor"] = self._get_vlm_processor()
parse_kwargs["storage"] = self.storage
parse_kwargs["_source_meta"] = {
"source_type": SourceType.HTTP,
"original_url": source,
}
parse_kwargs["original_source"] = source
explicit_name = kwargs.get("resource_name") or kwargs.get("source_name")
if explicit_name:
parse_kwargs["resource_name"] = _smart_stem(explicit_name)
return await run_stage(
"parse_artifact", self._get_parser_router().parse(source, **parse_kwargs)
)
# Phase 1: Accessor - get local resource. A caller may prepare a remote
# resource first so async routing uses the same detected file type.
if prepared_resource is not None:
local_resource = prepared_resource
else:
local_resource = await run_stage(
"source_prepare",
self.prepare(
source,
allow_local_path_resolution=allow_local_path_resolution,
**({"parse_mode": mode} if mode is ParseMode.NO_SPLIT else {}),
**kwargs,
),
)
# Use context manager for automatic cleanup, but preserve directories for TreeBuilder
try:
# Phase 2: Parser - parse the local resource
parse_kwargs = dict(kwargs)
# Source credentials are consumed by the accessor. Never forward
# them into parser kwargs, parse results, or later queue payloads.
parse_kwargs.pop("auth_config", None)
parse_kwargs.pop("github_token", None)
parse_kwargs.pop("tos_signature", None)
parse_kwargs.pop("tos_access", None)
parse_kwargs["instruction"] = instruction
parse_kwargs["_source_meta"] = local_resource.meta
parse_kwargs["resolved_extension"] = kwargs.get(
"resolved_extension"
) or local_resource.meta.get("resolved_extension", "")
# CRITICAL: Pass along the original_source!
# This is the full URL the user provided (e.g. "https://github.com/volcengine/OpenViking")
# CodeRepositoryParser and TreeBuilder need this to extract the org/repo format
# Without it, we'd only get the repo name without the org prefix!
parse_kwargs["original_source"] = local_resource.original_source
# Set resource_name from source_name or path
source_name = kwargs.get("source_name")
if source_name:
parse_kwargs["resource_name"] = _smart_stem(source_name)
parse_kwargs.setdefault("source_name", source_name)
else:
# For git repositories, use repo_name from meta if available
repo_name = local_resource.meta.get("repo_name")
if repo_name and local_resource.source_type == SourceType.GIT:
# Use the last part of repo_name as the resource_name (e.g., "OpenViking" from "volcengine/OpenViking")
parse_kwargs["resource_name"] = repo_name.split("/")[-1]
else:
# Prefer original_filename from meta for HTTP downloads
original_filename = local_resource.meta.get("original_filename")
if original_filename:
parse_kwargs.setdefault("resource_name", _smart_stem(original_filename))
parse_kwargs.setdefault("source_name", original_filename)
else:
parse_kwargs.setdefault("resource_name", _smart_stem(local_resource.path))
if mode is ParseMode.NO_SPLIT:
parse_kwargs["split_content"] = False
parse_kwargs["vlm_processor"] = self._get_vlm_processor()
parse_kwargs["storage"] = self.storage
# If it's a directory, use DirectoryParser which will delegate to CodeRepositoryParser if it's a git repo
if local_resource.path.is_dir():
from openviking.parse.parsers.directory import DirectoryParser
parser = DirectoryParser()
parse_kwargs["_feishu_import_plan"] = local_resource.feishu_plan
result = await run_stage(
"parse_artifact", parser.parse(str(local_resource.path), **parse_kwargs)
)
# Preserve temporary directory for TreeBuilder
if local_resource.is_temporary and not result.temp_dir_path:
result.temp_dir_path = str(local_resource.path)
# Mark as non-temporary so context manager doesn't clean it up
local_resource.is_temporary = False
return result
# For files, use ParserRouter to decide which parser to use
parser_router = self._get_parser_router()
if (
mode is ParseMode.NO_SPLIT
and local_resource.source_type == SourceType.FEISHU
and local_resource.meta.get("feishu_content_kind") == "file"
):
parse_kwargs["parser_backend"] = ParserBackend.INTERNAL
parse_kwargs.pop("feishu_access_token", None)
parse_kwargs.pop("lark_file", None)
checkpoint = parse_kwargs.pop("_feishu_checkpoint", None)
if checkpoint is not None and local_resource.meta.get("feishu_content_kind") == "file":
saved, save = checkpoint
if "file" in saved:
parse_kwargs["understanding_response_id"] = saved["file"]
async def record(response_id):
await save("file", response_id)
parse_kwargs["_response_checkpoint"] = record
return await run_stage(
"parse_artifact", parser_router.parse(local_resource, **parse_kwargs)
)
finally:
# Clean up temporary resources unless they need to be preserved
local_resource.cleanup()
def _is_url(self, source: str) -> bool:
"""Check if source is a URL."""
return is_remote_resource_source(source)
@staticmethod
def _has_prepared_understanding_response(kwargs: dict) -> bool:
from openviking.parse.understanding_api import (
PREPARED_FILE_ID_ARG,
PREPARED_RESPONSE_ID_ARG,
)
return any(
isinstance(kwargs.get(key), str) and bool(kwargs[key].strip())
for key in (PREPARED_RESPONSE_ID_ARG, PREPARED_FILE_ID_ARG)
)