Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

9.5 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Shared upload utilities for directory and file uploading to VikingFS."""

import asyncio
import os
from pathlib import Path
from typing import Any, List, Optional, Set, Tuple, Union

from openviking.parse.gitignore import GitignoreMatcher
from openviking.parse.output import (
    ARTIFACT_MANIFEST_NAME as ARTIFACT_MANIFEST_NAME,
)
from openviking.parse.output import (
    write_artifact_manifest,
)
from openviking.parse.parsers.constants import (
    ADDITIONAL_TEXT_EXTENSIONS,
    CODE_EXTENSIONS,
    DOCUMENTATION_EXTENSIONS,
    IGNORE_EXTENSIONS,
)
from openviking.parse.parsers.text_encoding import normalize_text_bytes
from openviking.utils.content_hash import content_md5
from openviking.utils.path_safety import sanitize_relative_viking_path
from openviking_cli.utils.logger import get_logger

logger = get_logger(__name__)


# Common text files that have no extension but should be treated as text.
_EXTENSIONLESS_TEXT_NAMES: Set[str] = {
    "README",
    "LICENSE",
    "LICENCE",
    "MAKEFILE",
    "DOCKERFILE",
    "VAGRANTFILE",
    "GEMFILE",
    "RAKEFILE",
    "PROCFILE",
    "CODEOWNERS",
    "AUTHORS",
    "CONTRIBUTORS",
    "CHANGELOG",
    "CHANGES",
    "NEWS",
    "NOTICE",
    "TODO",
    "BUILD",
}


def is_text_file(file_path: Union[str, Path]) -> bool:
    """Return True when the file extension is treated as text content."""
    p = Path(file_path)
    extension = p.suffix.lower()
    if extension:
        if (
            extension in CODE_EXTENSIONS
            or extension in DOCUMENTATION_EXTENSIONS
            or extension in ADDITIONAL_TEXT_EXTENSIONS
        ):
            return True

        from openviking.parse.parsers.code.ast.providers import supports_code_skeleton

        return supports_code_skeleton(str(p))
    # Extensionless files: check against known text file names (case-insensitive).
    return p.name.upper() in _EXTENSIONLESS_TEXT_NAMES


def detect_and_convert_encoding(content: bytes, file_path: Union[str, Path] = "") -> bytes:
    """Detect text encoding and normalize content to UTF-8 when needed."""
    if not is_text_file(file_path):
        return content
    return normalize_text_bytes(content, file_path)


def is_empty_file(file_path: Path) -> bool:
    """Return whether a file is empty or contains only ASCII whitespace."""
    with file_path.open("rb") as source:
        while chunk := source.read(8192):
            if chunk.strip():
                return False
    return True


def should_skip_file(
    file_path: Path,
    max_file_size: int = 10 * 1024 * 1024,
    ignore_extensions: Optional[Set[str]] = None,
) -> Tuple[bool, str]:
    """Return whether to skip a file and the reason for skipping."""
    effective_ignore_extensions = (
        ignore_extensions if ignore_extensions is not None else IGNORE_EXTENSIONS
    )

    if file_path.name.startswith("."):
        return True, "hidden file"

    if file_path.is_symlink():
        return True, "symbolic link"

    extension = file_path.suffix.lower()
    if extension in effective_ignore_extensions:
        return True, f"ignored extension: {extension}"

    try:
        file_size = file_path.stat().st_size
        if file_size > max_file_size:
            return True, f"file too large: {file_size} bytes"
        if is_empty_file(file_path):
            return True, "empty file"
    except OSError as exc:
        return True, f"os error: {exc}"

    return False, ""


_UPLOAD_CONCURRENCY = 8


async def upload_directory(
    local_dir: Path,
    base_rel: str,
    *,
    store: Any,
    artifact_ref: Any,
    ignore_dirs: Optional[Union[Set[str], List[str], str]] = None,
    ignore_extensions: Optional[Set[str]] = None,
    max_file_size: int = 10 * 1024 * 1024,
    include: Optional[str] = None,
    exclude: Optional[str] = None,
) -> Tuple[int, List[str]]:
    """Upload a directory into a parse output store, returning (count, warnings).

    Artifacts are written through the backend-agnostic :class:`ParseOutputStore`,
    so AGFS and local backends share one path with no branching. Files land under
    ``base_rel`` (e.g. ``repository``) as artifact-relative paths the store
    resolves to a local dir or an AGFS temp URI. An md5 manifest of the final
    (encoding-normalized) bytes is written at the artifact root so the incremental
    diff can compare fingerprints without re-reading file contents.
    """
    effective_ignore_extensions = (
        ignore_extensions if ignore_extensions is not None else IGNORE_EXTENSIONS
    )
    gitignore_matcher = GitignoreMatcher(local_dir)
    # Keep repository uploads aligned with DirectoryParser's public filtering
    # contract without duplicating its path/glob semantics.
    from openviking.parse.directory_scan import (
        _matches_exclude,
        _matches_include,
        _parse_patterns,
        _should_skip_directory,
    )

    if isinstance(ignore_dirs, str):
        parsed_ignore_dirs: Optional[Set[str]] = set(_parse_patterns(ignore_dirs)) or None
    elif ignore_dirs:
        parsed_ignore_dirs = {str(item) for item in ignore_dirs}
    else:
        parsed_ignore_dirs = None
    include_patterns = _parse_patterns(include)
    exclude_patterns = _parse_patterns(exclude)

    warnings: List[str] = []

    # --- Phase 1: Collect files and their artifact-relative parent dirs ---
    base = base_rel.strip("/")
    files_to_upload: List[Tuple[Path, str]] = []
    parent_dirs: Set[str] = set()

    for root, dirs, files in os.walk(local_dir):
        dir_path = Path(root)
        dir_spec = gitignore_matcher.spec_for_dir(dir_path)

        # Prune subdirectories in-place so os.walk won't descend into them
        kept = []
        for d in dirs:
            sub_dir_path = dir_path / d
            should_skip, _ = _should_skip_directory(
                sub_dir_path,
                local_dir,
                parsed_ignore_dirs,
            )
            if should_skip:
                continue

            if gitignore_matcher.is_ignored_dir(sub_dir_path, dir_spec):
                continue

            kept.append(d)

        dirs[:] = kept

        for file_name in files:
            file_path = dir_path / file_name
            should_skip, _ = should_skip_file(
                file_path,
                max_file_size=max_file_size,
                ignore_extensions=effective_ignore_extensions,
            )
            if should_skip:
                continue

            if gitignore_matcher.is_ignored_file(file_path, dir_spec):
                continue

            rel_path_str = str(file_path.relative_to(local_dir)).replace(os.sep, "/")
            if include_patterns and not _matches_include(file_name, include_patterns):
                continue
            if exclude_patterns and _matches_exclude(
                rel_path_str,
                file_name,
                exclude_patterns,
            ):
                continue
            # Artifact-relative path under the resource root; the store resolves it
            # to a local/agfs location.
            target = f"{base}/{rel_path_str}" if base else rel_path_str
            try:
                target = sanitize_relative_viking_path(target)
            except ValueError as exc:
                logger.warning(f"Skipping {file_path}: {exc}")
                continue
            files_to_upload.append((file_path, target))
            if "/" in target:
                parent_dirs.add(target.rsplit("/", 1)[0])

    # --- Phase 2: Pre-create unique parent dirs (shallowest first) ---
    # store.write_bytes also creates parents implicitly; this batches the unique
    # directories once to avoid redundant per-file creation on remote backends.
    for rel_dir in sorted(parent_dirs, key=lambda value: (value.count("/"), value)):
        try:
            await store.mkdir(artifact_ref, rel_dir)
        except Exception as e:
            logger.warning(f"Failed to create artifact directory {rel_dir}: {e}")

    # --- Phase 3: Upload files concurrently ---
    sem = asyncio.Semaphore(_UPLOAD_CONCURRENCY)
    errors: List[Optional[str]] = [None] * len(files_to_upload)
    # rel_path -> md5 of final bytes, collected for the manifest.
    md5_by_target: dict[str, str] = {}

    async def _upload_one(idx: int, file_path: Path, target: str) -> None:
        async with sem:

            def _read_and_encode() -> bytes:
                content = file_path.read_bytes()
                return detect_and_convert_encoding(content, file_path)

            try:
                encoded = await asyncio.to_thread(_read_and_encode)
                await store.write_bytes(artifact_ref, target, encoded)
                # Fingerprint the final (normalized) bytes at the write point; this
                # is a local read, no extra remote IO.
                md5_by_target[target] = content_md5(encoded)
            except Exception as exc:
                errors[idx] = f"Failed to upload {file_path}: {exc}"

    await asyncio.gather(*[_upload_one(i, fp, tgt) for i, (fp, tgt) in enumerate(files_to_upload)])

    for err in errors:
        if err:
            warnings.append(err)
            logger.warning(err)

    # Persist the md5 manifest at the artifact root so the incremental diff can
    # compare fingerprints without re-reading files. Only write it when every file
    # uploaded cleanly, so a partial manifest never masquerades as complete.
    if not any(errors):
        await write_artifact_manifest(store, artifact_ref, md5_by_target)

    uploaded_count = sum(1 for e in errors if e is None)
    return uploaded_count, warnings