Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(parse): preserve filters for local git repositories (#3190)

huangruitengcbcec52d7d36
diff --git a/openviking/parse/parsers/code/code.py b/openviking/parse/parsers/code/code.py
index 5efe4bbf8..89f2538f5 100644
--- a/openviking/parse/parsers/code/code.py
+++ b/openviking/parse/parsers/code/code.py
@@ -17,7 +17,7 @@ import time
import urllib.request
import zipfile
from pathlib import Path, PurePosixPath
-from typing import Any, List, Optional, Tuple, Union
+from typing import Any, List, Optional, Set, Tuple, Union
from urllib.parse import unquote, urlparse
from openviking.parse.base import (
@@ -170,7 +170,14 @@ class CodeRepositoryParser(BaseParser):
logger.info(f"Uploading to VikingFS: {target_root_uri}")
# 4. Upload to VikingFS (filtering on the fly)
- file_count = await self._upload_directory(local_dir, target_root_uri, viking_fs)
+ file_count = await self._upload_directory(
+ local_dir,
+ target_root_uri,
+ viking_fs,
+ ignore_dirs=kwargs.get("ignore_dirs"),
+ include=kwargs.get("include"),
+ exclude=kwargs.get("exclude"),
+ )
logger.info(f"Uploaded {file_count} files to {target_root_uri}")
@@ -572,7 +579,23 @@ class CodeRepositoryParser(BaseParser):
return name
- async def _upload_directory(self, local_dir: Path, viking_uri_base: str, viking_fs: Any) -> int:
+ async def _upload_directory(
+ self,
+ local_dir: Path,
+ viking_uri_base: str,
+ viking_fs: Any,
+ *,
+ ignore_dirs: Optional[Union[Set[str], List[str], str]] = None,
+ include: Optional[str] = None,
+ exclude: Optional[str] = None,
+ ) -> int:
"""Recursively upload directory to VikingFS using shared upload utilities."""
- count, _ = await upload_directory(local_dir, viking_uri_base, viking_fs)
+ count, _ = await upload_directory(
+ local_dir,
+ viking_uri_base,
+ viking_fs,
+ ignore_dirs=ignore_dirs,
+ include=include,
+ exclude=exclude,
+ )
return count
diff --git a/openviking/parse/parsers/upload_utils.py b/openviking/parse/parsers/upload_utils.py
index 54d5b09b2..e33de87c9 100644
--- a/openviking/parse/parsers/upload_utils.py
+++ b/openviking/parse/parsers/upload_utils.py
@@ -142,20 +142,38 @@ async def upload_directory(
local_dir: Path,
viking_uri_base: str,
viking_fs: Any,
- ignore_dirs: Optional[Set[str]] = None,
+ ignore_dirs: Optional[Union[Set[str], List[str], str]] = None,
ignore_extensions: Optional[Set[str]] = None,
max_file_size: int = 10 * 1024 * 1024,
+ include: Optional[str] = None,
+ exclude: Optional[str] = None,
) -> Tuple[int, List[str]]:
"""Upload an entire directory recursively and return uploaded count with warnings.
Optimized: collects all files in one pass, pre-creates directories upfront,
then uploads all files concurrently (up to _UPLOAD_CONCURRENCY at a time).
"""
- effective_ignore_dirs = ignore_dirs if ignore_dirs is not None else IGNORE_DIRS
effective_ignore_extensions = (
ignore_extensions if ignore_extensions is not None else IGNORE_EXTENSIONS
)
gitignore_matcher = GitignoreMatcher(local_dir)
+ # Keep repository uploads aligned with DirectoryParser's public filtering
+ # contract without duplicating its path/glob semantics.
+ from openviking.parse.directory_scan import (
+ _matches_exclude,
+ _matches_include,
+ _parse_patterns,
+ _should_skip_directory,
+ )
+
+ if isinstance(ignore_dirs, str):
+ parsed_ignore_dirs: Optional[Set[str]] = set(_parse_patterns(ignore_dirs)) or None
+ elif ignore_dirs:
+ parsed_ignore_dirs = {str(item) for item in ignore_dirs}
+ else:
+ parsed_ignore_dirs = None
+ include_patterns = _parse_patterns(include)
+ exclude_patterns = _parse_patterns(exclude)
warnings: List[str] = []
@@ -171,7 +189,11 @@ async def upload_directory(
kept = []
for d in dirs:
sub_dir_path = dir_path / d
- should_skip = should_skip_directory(d, ignore_dirs=effective_ignore_dirs)
+ should_skip, _ = _should_skip_directory(
+ sub_dir_path,
+ local_dir,
+ parsed_ignore_dirs,
+ )
if should_skip:
continue
@@ -196,6 +218,14 @@ async def upload_directory(
continue
rel_path_str = str(file_path.relative_to(local_dir)).replace(os.sep, "/")
+ if include_patterns and not _matches_include(file_name, include_patterns):
+ continue
+ if exclude_patterns and _matches_exclude(
+ rel_path_str,
+ file_name,
+ exclude_patterns,
+ ):
+ continue
try:
target_uri = safe_join_viking_uri(viking_uri_base, rel_path_str)
except ValueError as exc:
diff --git a/tests/parse/test_add_directory.py b/tests/parse/test_add_directory.py
index d5c7a3f1e..1b3f687e4 100644
--- a/tests/parse/test_add_directory.py
+++ b/tests/parse/test_add_directory.py
@@ -208,6 +208,37 @@ class TestDirectoryParserBasic:
p = DirectoryParser()
assert p.can_parse(tmp_path) is True
+ @pytest.mark.asyncio
+ async def test_git_repository_preserves_directory_filters(
+ self, tmp_path: Path, fake_fs: FakeVikingFS
+ ) -> None:
+ (tmp_path / ".git").mkdir()
+ (tmp_path / "notes").mkdir()
+ (tmp_path / "08_Attachments").mkdir()
+ (tmp_path / "notes" / "article.md").write_text("keep", encoding="utf-8")
+ (tmp_path / "notes" / "private.excalidraw.md").write_text(
+ "exclude", encoding="utf-8"
+ )
+ (tmp_path / "08_Attachments" / "diagram.md").write_text(
+ "ignore", encoding="utf-8"
+ )
+ (tmp_path / "main.py").write_text("exclude by include", encoding="utf-8")
+
+ with (
+ patch.object(BaseParser, "_get_viking_fs", return_value=fake_fs),
+ patch.object(DirectoryParser, "_add_git_metadata", new=AsyncMock()),
+ ):
+ result = await DirectoryParser().parse(
+ str(tmp_path),
+ ignore_dirs="08_Attachments",
+ include="*.md",
+ exclude="*.excalidraw.md",
+ )
+
+ uploaded_paths = {uri.split("/repository/", 1)[-1] for uri in fake_fs.files}
+ assert result.parser_name == "CodeRepositoryParser"
+ assert uploaded_paths == {"notes/article.md"}
+
def test_can_parse_file(self, tmp_path: Path):
f = tmp_path / "test.md"
f.write_text("hello")