Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(parse): extract shared upload utilities from CodeRepositoryParser (#87)

zemu9c1679f8cdde
diff --git a/openviking/parse/parsers/code/code.py b/openviking/parse/parsers/code/code.py
index de1788e90..8e774b433 100644
--- a/openviking/parse/parsers/code/code.py
+++ b/openviking/parse/parsers/code/code.py
@@ -24,8 +24,6 @@ from openviking.parse.base import (
create_parse_result,
)
from openviking.parse.parsers.base_parser import BaseParser
-from openviking.utils.logger import get_logger
-
from openviking.parse.parsers.constants import (
CODE_EXTENSIONS,
DOCUMENTATION_EXTENSIONS,
@@ -34,10 +32,9 @@ from openviking.parse.parsers.constants import (
FILE_TYPE_OTHER,
IGNORE_DIRS,
IGNORE_EXTENSIONS,
- ADDITIONAL_TEXT_EXTENSIONS,
- TEXT_ENCODINGS,
- UTF8_VARIANTS,
)
+from openviking.parse.parsers.upload_utils import upload_directory
+from openviking.utils.logger import get_logger
logger = get_logger(__name__)
@@ -254,110 +251,6 @@ class CodeRepositoryParser(BaseParser):
return name
async def _upload_directory(self, local_dir: Path, viking_uri_base: str, viking_fs: Any) -> int:
- """
- Recursively upload directory to VikingFS.
-
- Args:
- local_dir: Local source directory
- viking_uri_base: Target Viking URI
- viking_fs: VikingFS instance
-
- Returns:
- Number of uploaded files
- """
- count = 0
-
- # Ensure target directory exists (although write_file handles parents, mkdir ensures root exists)
- await viking_fs.mkdir(viking_uri_base, exist_ok=True)
-
- for root, dirs, files in os.walk(local_dir):
- # Modify dirs in-place to skip ignored directories
- dirs[:] = [d for d in dirs if d not in IGNORE_DIRS and not d.startswith(".")]
-
- for file in files:
- if file.startswith("."):
- continue
-
- file_path = Path(root) / file
-
- file_path = Path(root) / file
-
- # Calculate relative path for URI construction
- rel_path = file_path.relative_to(local_dir)
- rel_path_str = str(rel_path).replace(os.sep, "/")
-
- # Check if it's a symbolic link (skip and log)
- if os.path.islink(file_path):
- target = os.readlink(file_path)
- logger.info(f"Ignoring symbolic link {rel_path_str}: {file_path} -> {target}")
- continue
-
- # Check extension
- if file_path.suffix.lower() in IGNORE_EXTENSIONS:
- continue
-
- # Check file size (skip > 10MB)
- try:
- size = file_path.stat().st_size
- if size > 10 * 1024 * 1024:
- logger.warning(f"Skipping large file {file}: {size} bytes")
- continue
- if size == 0:
- continue
- except OSError:
- continue
-
- # Construct Viking URI: base + rel_path
- target_uri = f"{viking_uri_base}/{rel_path_str}"
-
- # Read and upload
- try:
- content = file_path.read_bytes()
-
- # Check if this is a text file that might need encoding conversion
- extension = file_path.suffix.lower()
- is_text_file = (
- extension in CODE_EXTENSIONS
- or extension in DOCUMENTATION_EXTENSIONS
- or extension in ADDITIONAL_TEXT_EXTENSIONS
- )
-
- if is_text_file:
- # Try to detect encoding and convert to UTF-8
- try:
- detected_encoding = None
- for encoding in TEXT_ENCODINGS:
- try:
- # Try to decode with this encoding
- decoded = content.decode(encoding)
- detected_encoding = encoding
- break
- except UnicodeDecodeError:
- continue
-
- if detected_encoding and detected_encoding not in UTF8_VARIANTS:
- # Convert to UTF-8
- decoded = content.decode(detected_encoding, errors="replace")
- content = decoded.encode("utf-8")
- logger.debug(
- f"Converted {rel_path_str} from {detected_encoding} to UTF-8"
- )
-
- except Exception as encode_error:
- # If encoding detection/conversion fails, use original bytes
- logger.warning(
- f"Encoding detection failed for {file_path}: {encode_error}"
- )
-
- # Use write_file_bytes for safety
- await viking_fs.write_file_bytes(target_uri, content)
-
- # TODO: Add metadata tagging when VikingFS supports it
- # file_type = self._detect_file_type(file_path)
- # await viking_fs.set_metadata(target_uri, {"file_type": file_type})
-
- count += 1
- except Exception as e:
- logger.warning(f"Failed to upload {file_path}: {e}")
-
+ """Recursively upload directory to VikingFS using shared upload utilities."""
+ count, _ = await upload_directory(local_dir, viking_uri_base, viking_fs)
return count
diff --git a/openviking/parse/parsers/upload_utils.py b/openviking/parse/parsers/upload_utils.py
new file mode 100644
index 000000000..06e28a4cb
--- /dev/null
+++ b/openviking/parse/parsers/upload_utils.py
@@ -0,0 +1,229 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Shared upload utilities for directory and file uploading to VikingFS."""
+
+import os
+import re
+from pathlib import Path
+from typing import Any, List, Optional, Set, Tuple, Union
+
+from openviking.parse.parsers.constants import (
+ ADDITIONAL_TEXT_EXTENSIONS,
+ CODE_EXTENSIONS,
+ DOCUMENTATION_EXTENSIONS,
+ IGNORE_DIRS,
+ IGNORE_EXTENSIONS,
+ TEXT_ENCODINGS,
+ UTF8_VARIANTS,
+)
+from openviking.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+# Common text files that have no extension but should be treated as text.
+_EXTENSIONLESS_TEXT_NAMES: Set[str] = {
+ "LICENSE",
+ "LICENCE",
+ "MAKEFILE",
+ "DOCKERFILE",
+ "VAGRANTFILE",
+ "GEMFILE",
+ "RAKEFILE",
+ "PROCFILE",
+ "CODEOWNERS",
+ "AUTHORS",
+ "CONTRIBUTORS",
+ "CHANGELOG",
+ "CHANGES",
+ "NEWS",
+ "NOTICE",
+ "TODO",
+}
+
+
+def is_text_file(file_path: Union[str, Path]) -> bool:
+ """Return True when the file extension is treated as text content."""
+ p = Path(file_path)
+ extension = p.suffix.lower()
+ if extension:
+ return (
+ extension in CODE_EXTENSIONS
+ or extension in DOCUMENTATION_EXTENSIONS
+ or extension in ADDITIONAL_TEXT_EXTENSIONS
+ )
+ # Extensionless files: check against known text file names (case-insensitive).
+ return p.name.upper() in _EXTENSIONLESS_TEXT_NAMES
+
+
+def detect_and_convert_encoding(content: bytes, file_path: Union[str, Path] = "") -> bytes:
+ """Detect text encoding and normalize content to UTF-8 when needed."""
+ if not is_text_file(file_path):
+ return content
+
+ try:
+ detected_encoding: Optional[str] = None
+ for encoding in TEXT_ENCODINGS:
+ try:
+ content.decode(encoding)
+ detected_encoding = encoding
+ break
+ except UnicodeDecodeError:
+ continue
+
+ if detected_encoding is None:
+ logger.warning(f"Encoding detection failed for {file_path}: no matching encoding found")
+ return content
+
+ if detected_encoding not in UTF8_VARIANTS:
+ decoded_content = content.decode(detected_encoding, errors="replace")
+ content = decoded_content.encode("utf-8")
+ logger.debug(f"Converted {file_path} from {detected_encoding} to UTF-8")
+
+ return content
+ except Exception as exc:
+ logger.warning(f"Encoding detection failed for {file_path}: {exc}")
+ return content
+
+
+def should_skip_file(
+ file_path: Path,
+ max_file_size: int = 10 * 1024 * 1024,
+ ignore_extensions: Optional[Set[str]] = None,
+) -> Tuple[bool, str]:
+ """Return whether to skip a file and the reason for skipping."""
+ effective_ignore_extensions = (
+ ignore_extensions if ignore_extensions is not None else IGNORE_EXTENSIONS
+ )
+
+ if file_path.name.startswith("."):
+ return True, "hidden file"
+
+ if file_path.is_symlink():
+ return True, "symbolic link"
+
+ extension = file_path.suffix.lower()
+ if extension in effective_ignore_extensions:
+ return True, f"ignored extension: {extension}"
+
+ try:
+ file_size = file_path.stat().st_size
+ if file_size > max_file_size:
+ return True, f"file too large: {file_size} bytes"
+ if file_size == 0:
+ return True, "empty file"
+ except OSError as exc:
+ return True, f"os error: {exc}"
+
+ return False, ""
+
+
+def should_skip_directory(
+ dir_name: str,
+ ignore_dirs: Optional[Set[str]] = None,
+) -> bool:
+ """Return True when a directory should be skipped during traversal."""
+ effective_ignore_dirs = ignore_dirs if ignore_dirs is not None else IGNORE_DIRS
+ return dir_name in effective_ignore_dirs or dir_name.startswith(".")
+
+
+_UNSAFE_PATH_RE = re.compile(r"(^|[\\/])\.\.($|[\\/])")
+_DRIVE_RE = re.compile(r"^[A-Za-z]:")
+
+
+def _sanitize_rel_path(rel_path: str) -> str:
+ """Normalize a relative path and reject unsafe components.
+
+ Uses OS-independent checks so that Windows-style drive prefixes and
+ backslash separators are rejected even when running on Linux/macOS.
+ """
+ if not rel_path:
+ raise ValueError(f"Unsafe relative path rejected: {rel_path!r}")
+ # Reject absolute paths (Unix or Windows style)
+ if rel_path.startswith("/") or rel_path.startswith("\\"):
+ raise ValueError(f"Unsafe relative path rejected: {rel_path}")
+ # Reject Windows drive letters (C:\..., C:foo)
+ if _DRIVE_RE.match(rel_path):
+ raise ValueError(f"Unsafe relative path rejected: {rel_path}")
+ # Reject parent-directory traversal (../ or ..\)
+ if _UNSAFE_PATH_RE.search(rel_path):
+ raise ValueError(f"Unsafe relative path rejected: {rel_path}")
+ # Normalize to forward slashes
+ return rel_path.replace("\\", "/")
+
+
+async def upload_text_files(
+ file_paths: List[Tuple[Path, str]],
+ viking_uri_base: str,
+ viking_fs: Any,
+) -> Tuple[int, List[str]]:
+ """Upload text files to VikingFS and return uploaded count with warnings."""
+ uploaded_count = 0
+ warnings: List[str] = []
+
+ for file_path, rel_path in file_paths:
+ try:
+ safe_rel = _sanitize_rel_path(rel_path)
+ target_uri = f"{viking_uri_base}/{safe_rel}"
+ content = file_path.read_bytes()
+ content = detect_and_convert_encoding(content, file_path)
+ await viking_fs.write_file_bytes(target_uri, content)
+ uploaded_count += 1
+ except Exception as exc:
+ warning = f"Failed to upload {file_path}: {exc}"
+ warnings.append(warning)
+ logger.warning(warning)
+
+ return uploaded_count, warnings
+
+
+async def upload_directory(
+ local_dir: Path,
+ viking_uri_base: str,
+ viking_fs: Any,
+ ignore_dirs: Optional[Set[str]] = None,
+ ignore_extensions: Optional[Set[str]] = None,
+ max_file_size: int = 10 * 1024 * 1024,
+) -> Tuple[int, List[str]]:
+ """Upload an entire directory recursively and return uploaded count with warnings."""
+ effective_ignore_dirs = ignore_dirs if ignore_dirs is not None else IGNORE_DIRS
+ effective_ignore_extensions = (
+ ignore_extensions if ignore_extensions is not None else IGNORE_EXTENSIONS
+ )
+
+ uploaded_count = 0
+ warnings: List[str] = []
+
+ await viking_fs.mkdir(viking_uri_base, exist_ok=True)
+
+ for root, dirs, files in os.walk(local_dir):
+ dirs[:] = [
+ dir_name
+ for dir_name in dirs
+ if not should_skip_directory(dir_name, ignore_dirs=effective_ignore_dirs)
+ ]
+
+ for file_name in files:
+ file_path = Path(root) / file_name
+ should_skip, _ = should_skip_file(
+ file_path,
+ max_file_size=max_file_size,
+ ignore_extensions=effective_ignore_extensions,
+ )
+ if should_skip:
+ continue
+
+ rel_path_str = str(file_path.relative_to(local_dir)).replace(os.sep, "/")
+ try:
+ safe_rel = _sanitize_rel_path(rel_path_str)
+ target_uri = f"{viking_uri_base}/{safe_rel}"
+ content = file_path.read_bytes()
+ content = detect_and_convert_encoding(content, file_path)
+ await viking_fs.write_file_bytes(target_uri, content)
+ uploaded_count += 1
+ except Exception as exc:
+ warning = f"Failed to upload {file_path}: {exc}"
+ warnings.append(warning)
+ logger.warning(warning)
+
+ return uploaded_count, warnings
diff --git a/tests/test_upload_utils.py b/tests/test_upload_utils.py
new file mode 100644
index 000000000..f334a5839
--- /dev/null
+++ b/tests/test_upload_utils.py
@@ -0,0 +1,508 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Tests for shared upload utilities."""
+
+from pathlib import Path
+from typing import Dict, List
+
+import pytest
+
+from openviking.parse.parsers.upload_utils import (
+ _sanitize_rel_path,
+ detect_and_convert_encoding,
+ is_text_file,
+ should_skip_directory,
+ should_skip_file,
+ upload_directory,
+ upload_text_files,
+)
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+class FakeVikingFS:
+ """Minimal VikingFS mock for testing upload functions."""
+
+ def __init__(self) -> None:
+ self.files: Dict[str, bytes] = {}
+ self.dirs: List[str] = []
+
+ async def write_file_bytes(self, uri: str, content: bytes) -> None:
+ self.files[uri] = content
+
+ async def mkdir(self, uri: str, exist_ok: bool = False) -> None:
+ self.dirs.append(uri)
+
+
+def viking_fs() -> FakeVikingFS:
+ return FakeVikingFS()
+
+
+def tmp_dir(tmp_path: Path) -> Path:
+ """Create a temporary directory with sample files for testing."""
+ # Text files
+ (tmp_path / "hello.py").write_text("print('hello')", encoding="utf-8")
+ (tmp_path / "readme.md").write_text("# README", encoding="utf-8")
+ (tmp_path / "config.yaml").write_text("key: value", encoding="utf-8")
+
+ # Hidden file
+ (tmp_path / ".hidden").write_text("secret", encoding="utf-8")
+
+ # Binary-extension file
+ (tmp_path / "image.png").write_bytes(b"\x89PNG\r\n")
+
+ # Empty file
+ (tmp_path / "empty.txt").write_bytes(b"")
+
+ # Subdirectory
+ sub = tmp_path / "src"
+ sub.mkdir()
+ (sub / "main.go").write_text("package main", encoding="utf-8")
+
+ # Ignored directory
+ pycache = tmp_path / "__pycache__"
+ pycache.mkdir()
+ (pycache / "mod.pyc").write_bytes(b"\x00\x00")
+
+ return tmp_path
+
+
+# ---------------------------------------------------------------------------
+# is_text_file
+# ---------------------------------------------------------------------------
+
+
+class TestIsTextFile:
+ def test_code_extensions(self) -> None:
+ assert is_text_file("main.py") is True
+ assert is_text_file("app.js") is True
+ assert is_text_file("lib.go") is True
+
+ def test_documentation_extensions(self) -> None:
+ assert is_text_file("README.md") is True
+ assert is_text_file("notes.txt") is True
+ assert is_text_file("guide.rst") is True
+
+ def test_additional_text_extensions(self) -> None:
+ assert is_text_file("settings.ini") is True
+ assert is_text_file("data.csv") is True
+
+ def test_non_text_extensions(self) -> None:
+ assert is_text_file("photo.png") is False
+ assert is_text_file("video.mp4") is False
+ assert is_text_file("archive.zip") is False
+ assert is_text_file("program.exe") is False
+
+ def test_no_extension_known_names(self) -> None:
+ assert is_text_file("Makefile") is True
+ assert is_text_file("LICENSE") is True
+ assert is_text_file("Dockerfile") is True
+
+ def test_no_extension_unknown_names(self) -> None:
+ assert is_text_file("randomfile") is False
+
+ def test_no_extension_case_insensitive(self) -> None:
+ assert is_text_file("makefile") is True
+ assert is_text_file("license") is True
+ assert is_text_file("dockerfile") is True
+
+ def test_case_insensitive(self) -> None:
+ assert is_text_file("MAIN.PY") is True
+ assert is_text_file("README.MD") is True
+
+
+# ---------------------------------------------------------------------------
+# detect_and_convert_encoding
+# ---------------------------------------------------------------------------
+
+
+class TestDetectAndConvertEncoding:
+ def test_utf8_passthrough(self) -> None:
+ content = "hello world".encode("utf-8")
+ result = detect_and_convert_encoding(content, "test.py")
+ assert result == content
+
+ def test_gbk_to_utf8(self) -> None:
+ text = "你好世界"
+ content = text.encode("gbk")
+ result = detect_and_convert_encoding(content, "test.py")
+ assert result.decode("utf-8") == text
+
+ def test_non_text_file_passthrough(self) -> None:
+ content = b"\x89PNG\r\n\x1a\n"
+ result = detect_and_convert_encoding(content, "image.png")
+ assert result == content
+
+ def test_empty_file_path(self) -> None:
+ content = b"hello"
+ result = detect_and_convert_encoding(content, "")
+ # Empty path has no extension, so is_text_file returns False
+ assert result == content
+
+ def test_latin1_to_utf8(self) -> None:
+ text = "café"
+ content = text.encode("latin-1")
+ result = detect_and_convert_encoding(content, "test.txt")
+ assert "caf" in result.decode("utf-8")
+
+
+# ---------------------------------------------------------------------------
+# should_skip_file
+# ---------------------------------------------------------------------------
+
+
+class TestShouldSkipFile:
+ def test_hidden_file(self, tmp_path: Path) -> None:
+ f = tmp_path / ".gitignore"
+ f.write_text("node_modules", encoding="utf-8")
+ skip, reason = should_skip_file(f)
+ assert skip is True
+ assert "hidden" in reason
+
+ def test_ignored_extension(self, tmp_path: Path) -> None:
+ f = tmp_path / "photo.jpg"
+ f.write_bytes(b"\xff\xd8\xff")
+ skip, reason = should_skip_file(f)
+ assert skip is True
+ assert ".jpg" in reason
+
+ def test_large_file(self, tmp_path: Path) -> None:
+ f = tmp_path / "big.txt"
+ f.write_bytes(b"x" * 100)
+ skip, reason = should_skip_file(f, max_file_size=50)
+ assert skip is True
+ assert "too large" in reason
+
+ def test_empty_file(self, tmp_path: Path) -> None:
+ f = tmp_path / "empty.py"
+ f.write_bytes(b"")
+ skip, reason = should_skip_file(f)
+ assert skip is True
+ assert "empty" in reason
+
+ def test_normal_file(self, tmp_path: Path) -> None:
+ f = tmp_path / "main.py"
+ f.write_text("print(1)", encoding="utf-8")
+ skip, reason = should_skip_file(f)
+ assert skip is False
+ assert reason == ""
+
+ def test_custom_ignore_extensions(self, tmp_path: Path) -> None:
+ f = tmp_path / "data.csv"
+ f.write_text("a,b,c", encoding="utf-8")
+ skip, _ = should_skip_file(f, ignore_extensions={".csv"})
+ assert skip is True
+
+ def test_symlink(self, tmp_path: Path) -> None:
+ target = tmp_path / "real.txt"
+ target.write_text("content", encoding="utf-8")
+ link = tmp_path / "link.txt"
+ try:
+ link.symlink_to(target)
+ except OSError:
+ pytest.skip("Symlinks not supported on this platform")
+ skip, reason = should_skip_file(link)
+ assert skip is True
+ assert "symbolic" in reason
+
+
+# ---------------------------------------------------------------------------
+# should_skip_directory
+# ---------------------------------------------------------------------------
+
+
+class TestShouldSkipDirectory:
+ def test_ignored_dirs(self) -> None:
+ assert should_skip_directory(".git") is True
+ assert should_skip_directory("__pycache__") is True
+ assert should_skip_directory("node_modules") is True
+
+ def test_hidden_dirs(self) -> None:
+ assert should_skip_directory(".vscode") is True
+ assert should_skip_directory(".idea") is True
+
+ def test_normal_dirs(self) -> None:
+ assert should_skip_directory("src") is False
+ assert should_skip_directory("tests") is False
+ assert should_skip_directory("docs") is False
+
+
+# ---------------------------------------------------------------------------
+# upload_text_files
+# ---------------------------------------------------------------------------
+
+
+class TestUploadTextFiles:
+ @pytest.mark.asyncio
+ async def test_upload_success(self, tmp_path: Path, viking_fs: FakeVikingFS) -> None:
+ f = tmp_path / "hello.py"
+ f.write_text("print('hi')", encoding="utf-8")
+ file_paths = [(f, "hello.py")]
+
+ count, warnings = await upload_text_files(file_paths, "viking://temp/abc", viking_fs)
+
+ assert count == 1
+ assert len(warnings) == 0
+ assert "viking://temp/abc/hello.py" in viking_fs.files
+
+ @pytest.mark.asyncio
+ async def test_upload_multiple(self, tmp_path: Path, viking_fs: FakeVikingFS) -> None:
+ f1 = tmp_path / "a.py"
+ f1.write_text("a", encoding="utf-8")
+ f2 = tmp_path / "b.md"
+ f2.write_text("b", encoding="utf-8")
+ file_paths = [(f1, "a.py"), (f2, "b.md")]
+
+ count, warnings = await upload_text_files(file_paths, "viking://temp/x", viking_fs)
+
+ assert count == 2
+ assert len(warnings) == 0
+
+ @pytest.mark.asyncio
+ async def test_upload_with_encoding_conversion(
+ self, tmp_path: Path, viking_fs: FakeVikingFS
+ ) -> None:
+ f = tmp_path / "chinese.py"
+ f.write_bytes("你好".encode("gbk"))
+ file_paths = [(f, "chinese.py")]
+
+ count, warnings = await upload_text_files(file_paths, "viking://temp/enc", viking_fs)
+
+ assert count == 1
+ uploaded = viking_fs.files["viking://temp/enc/chinese.py"]
+ assert uploaded.decode("utf-8") == "你好"
+
+ @pytest.mark.asyncio
+ async def test_upload_nonexistent_file(self, tmp_path: Path, viking_fs: FakeVikingFS) -> None:
+ fake = tmp_path / "nonexistent.py"
+ file_paths = [(fake, "nonexistent.py")]
+
+ count, warnings = await upload_text_files(file_paths, "viking://temp/err", viking_fs)
+
+ assert count == 0
+ assert len(warnings) == 1
+
+
+# ---------------------------------------------------------------------------
+# upload_directory
+# ---------------------------------------------------------------------------
+
+
+class TestUploadDirectory:
+ @pytest.mark.asyncio
+ async def test_basic_upload(self, tmp_dir: Path, viking_fs: FakeVikingFS) -> None:
+ count, warnings = await upload_directory(tmp_dir, "viking://temp/test", viking_fs)
+
+ # Should upload: hello.py, readme.md, config.yaml, src/main.go
+ # Should skip: .hidden, image.png, empty.txt, __pycache__/mod.pyc
+ assert count == 4
+ assert "viking://temp/test/hello.py" in viking_fs.files
+ assert "viking://temp/test/readme.md" in viking_fs.files
+ assert "viking://temp/test/config.yaml" in viking_fs.files
+ assert "viking://temp/test/src/main.go" in viking_fs.files
+
+ @pytest.mark.asyncio
+ async def test_skips_hidden_files(self, tmp_dir: Path, viking_fs: FakeVikingFS) -> None:
+ await upload_directory(tmp_dir, "viking://temp/test", viking_fs)
+ assert all(".hidden" not in uri for uri in viking_fs.files)
+
+ @pytest.mark.asyncio
+ async def test_skips_ignored_dirs(self, tmp_dir: Path, viking_fs: FakeVikingFS) -> None:
+ await upload_directory(tmp_dir, "viking://temp/test", viking_fs)
+ assert all("__pycache__" not in uri for uri in viking_fs.files)
+
+ @pytest.mark.asyncio
+ async def test_skips_ignored_extensions(self, tmp_dir: Path, viking_fs: FakeVikingFS) -> None:
+ await upload_directory(tmp_dir, "viking://temp/test", viking_fs)
+ assert all(".png" not in uri for uri in viking_fs.files)
+
+ @pytest.mark.asyncio
+ async def test_skips_empty_files(self, tmp_dir: Path, viking_fs: FakeVikingFS) -> None:
+ await upload_directory(tmp_dir, "viking://temp/test", viking_fs)
+ assert all("empty.txt" not in uri for uri in viking_fs.files)
+
+ @pytest.mark.asyncio
+ async def test_creates_root_dir(self, tmp_dir: Path, viking_fs: FakeVikingFS) -> None:
+ await upload_directory(tmp_dir, "viking://temp/root", viking_fs)
+ assert "viking://temp/root" in viking_fs.dirs
+
+ @pytest.mark.asyncio
+ async def test_custom_ignore_dirs(self, tmp_dir: Path, viking_fs: FakeVikingFS) -> None:
+ count, _ = await upload_directory(
+ tmp_dir, "viking://temp/test", viking_fs, ignore_dirs={"src"}
+ )
+ assert all("src/" not in uri for uri in viking_fs.files)
+ # Positive assertion: non-ignored files should still be uploaded
+ assert count > 0
+ assert "viking://temp/test/hello.py" in viking_fs.files
+
+ @pytest.mark.asyncio
+ async def test_custom_max_file_size(self, tmp_dir: Path, viking_fs: FakeVikingFS) -> None:
+ count, _ = await upload_directory(tmp_dir, "viking://temp/test", viking_fs, max_file_size=5)
+ # Most files are > 5 bytes, so fewer uploads
+ assert count < 4
+
+
+# ---------------------------------------------------------------------------
+# detect_and_convert_encoding (additional edge cases)
+# ---------------------------------------------------------------------------
+
+
+class TestDetectAndConvertEncodingEdgeCases:
+ def test_extensionless_text_file_encoding(self) -> None:
+ text = "你好世界"
+ content = text.encode("gbk")
+ result = detect_and_convert_encoding(content, "LICENSE")
+ # LICENSE is now recognized as text, so encoding conversion should happen
+ assert result.decode("utf-8") == text
+
+ def test_undecodable_content(self) -> None:
+ # Note: TEXT_ENCODINGS includes iso-8859-1 which can decode any byte sequence,
+ # so the "no matching encoding" branch is effectively unreachable.
+ # This test verifies that arbitrary bytes are handled gracefully regardless.
+ content = bytes(range(128, 256)) * 10
+ result = detect_and_convert_encoding(content, "test.py")
+ assert isinstance(result, bytes)
+
+
+# ---------------------------------------------------------------------------
+# should_skip_file (additional edge cases)
+# ---------------------------------------------------------------------------
+
+
+class TestShouldSkipFileEdgeCases:
+ def test_oserror_on_stat(self, tmp_path: Path) -> None:
+ f = tmp_path / "ghost.py"
+ # File doesn't exist, stat() will raise OSError
+ skip, reason = should_skip_file(f)
+ assert skip is True
+ assert "os error" in reason
+
+
+# ---------------------------------------------------------------------------
+# should_skip_directory (custom ignore_dirs)
+# ---------------------------------------------------------------------------
+
+
+class TestShouldSkipDirectoryCustom:
+ def test_custom_ignore_dirs(self) -> None:
+ assert should_skip_directory("vendor", ignore_dirs={"vendor"}) is True
+ assert should_skip_directory("src", ignore_dirs={"vendor"}) is False
+
+ def test_hidden_dir_with_custom_ignore(self) -> None:
+ # Hidden dirs should still be skipped even with custom ignore set
+ assert should_skip_directory(".secret", ignore_dirs={"vendor"}) is True
+
+
+# ---------------------------------------------------------------------------
+# _sanitize_rel_path (path traversal protection)
+# ---------------------------------------------------------------------------
+
+
+class TestSanitizeRelPath:
+ def test_normal_path(self) -> None:
+ assert _sanitize_rel_path("src/main.py") == "src/main.py"
+
+ def test_rejects_parent_traversal(self) -> None:
+ with pytest.raises(ValueError, match="Unsafe"):
+ _sanitize_rel_path("../etc/passwd")
+
+ def test_rejects_absolute_path(self) -> None:
+ with pytest.raises(ValueError, match="Unsafe"):
+ _sanitize_rel_path("/etc/passwd")
+
+ def test_rejects_windows_drive_absolute(self) -> None:
+ with pytest.raises(ValueError, match="Unsafe"):
+ _sanitize_rel_path("C:\\Windows\\System32")
+
+ def test_rejects_windows_drive_relative(self) -> None:
+ with pytest.raises(ValueError, match="Unsafe"):
+ _sanitize_rel_path("C:Windows\\System32")
+
+ def test_rejects_nested_traversal(self) -> None:
+ with pytest.raises(ValueError, match="Unsafe"):
+ _sanitize_rel_path("foo/../../bar")
+
+ def test_normalizes_backslashes(self) -> None:
+ result = _sanitize_rel_path("src\\main.py")
+ assert result == "src/main.py"
+
+
+# ---------------------------------------------------------------------------
+# upload_text_files (additional edge cases)
+# ---------------------------------------------------------------------------
+
+
+class TestUploadTextFilesEdgeCases:
+ @pytest.mark.asyncio
+ async def test_rejects_path_traversal(self, tmp_path: Path, viking_fs: FakeVikingFS) -> None:
+ f = tmp_path / "evil.py"
+ f.write_text("hack", encoding="utf-8")
+ file_paths = [(f, "../../../etc/passwd")]
+
+ count, warnings = await upload_text_files(file_paths, "viking://temp/safe", viking_fs)
+
+ assert count == 0
+ assert len(warnings) == 1
+
+ @pytest.mark.asyncio
+ async def test_upload_failure_produces_warning(self, tmp_path: Path) -> None:
+ class FailingFS:
+ async def write_file_bytes(self, uri: str, content: bytes) -> None:
+ raise IOError("disk full")
+
+ async def mkdir(self, uri: str, exist_ok: bool = False) -> None:
+ pass
+
+ f = tmp_path / "ok.py"
+ f.write_text("print(1)", encoding="utf-8")
+ file_paths = [(f, "ok.py")]
+
+ count, warnings = await upload_text_files(file_paths, "viking://temp/fail", FailingFS())
+
+ assert count == 0
+ assert len(warnings) == 1
+ assert "disk full" in warnings[0]
+
+
+# ---------------------------------------------------------------------------
+# upload_directory (additional edge cases)
+# ---------------------------------------------------------------------------
+
+
+class TestUploadDirectoryEdgeCases:
+ @pytest.mark.asyncio
+ async def test_write_failure_produces_warning(self, tmp_path: Path) -> None:
+ class FailingWriteFS:
+ async def write_file_bytes(self, uri: str, content: bytes) -> None:
+ raise IOError("write error")
+
+ async def mkdir(self, uri: str, exist_ok: bool = False) -> None:
+ pass
+
+ (tmp_path / "ok.py").write_text("print(1)", encoding="utf-8")
+
+ count, warnings = await upload_directory(tmp_path, "viking://temp/fail", FailingWriteFS())
+
+ assert count == 0
+ assert len(warnings) == 1
+ assert "write error" in warnings[0]
+
+
+# ---------------------------------------------------------------------------
+# _sanitize_rel_path (additional edge cases)
+# ---------------------------------------------------------------------------
+
+
+class TestSanitizeRelPathEdgeCases:
+ def test_rejects_empty_path(self) -> None:
+ with pytest.raises(ValueError, match="Unsafe"):
+ _sanitize_rel_path("")
+
+ def test_rejects_backslash_absolute(self) -> None:
+ with pytest.raises(ValueError, match="Unsafe"):
+ _sanitize_rel_path("\\Windows\\System32")