Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(parse): preserve filters for local git repositories (#3190)
huangruitengcbcec52d7d36
diff --git a/openviking/parse/parsers/code/code.py b/openviking/parse/parsers/code/code.pyindex 5efe4bbf8..89f2538f5 100644--- a/openviking/parse/parsers/code/code.py+++ b/openviking/parse/parsers/code/code.py@@ -17,7 +17,7 @@ import timeimport urllib.requestimport zipfilefrom pathlib import Path, PurePosixPath-from typing import Any, List, Optional, Tuple, Union+from typing import Any, List, Optional, Set, Tuple, Unionfrom urllib.parse import unquote, urlparsefrom openviking.parse.base import (@@ -170,7 +170,14 @@ class CodeRepositoryParser(BaseParser):logger.info(f"Uploading to VikingFS: {target_root_uri}")# 4. Upload to VikingFS (filtering on the fly)- file_count = await self._upload_directory(local_dir, target_root_uri, viking_fs)+ file_count = await self._upload_directory(+ local_dir,+ target_root_uri,+ viking_fs,+ ignore_dirs=kwargs.get("ignore_dirs"),+ include=kwargs.get("include"),+ exclude=kwargs.get("exclude"),+ )logger.info(f"Uploaded {file_count} files to {target_root_uri}")@@ -572,7 +579,23 @@ class CodeRepositoryParser(BaseParser):return name- async def _upload_directory(self, local_dir: Path, viking_uri_base: str, viking_fs: Any) -> int:+ async def _upload_directory(+ self,+ local_dir: Path,+ viking_uri_base: str,+ viking_fs: Any,+ *,+ ignore_dirs: Optional[Union[Set[str], List[str], str]] = None,+ include: Optional[str] = None,+ exclude: Optional[str] = None,+ ) -> int:"""Recursively upload directory to VikingFS using shared upload utilities."""- count, _ = await upload_directory(local_dir, viking_uri_base, viking_fs)+ count, _ = await upload_directory(+ local_dir,+ viking_uri_base,+ viking_fs,+ ignore_dirs=ignore_dirs,+ include=include,+ exclude=exclude,+ )return countdiff --git a/openviking/parse/parsers/upload_utils.py b/openviking/parse/parsers/upload_utils.pyindex 54d5b09b2..e33de87c9 100644--- a/openviking/parse/parsers/upload_utils.py+++ b/openviking/parse/parsers/upload_utils.py@@ -142,20 +142,38 @@ async def upload_directory(local_dir: Path,viking_uri_base: str,viking_fs: Any,- ignore_dirs: Optional[Set[str]] = None,+ ignore_dirs: Optional[Union[Set[str], List[str], str]] = None,ignore_extensions: Optional[Set[str]] = None,max_file_size: int = 10 * 1024 * 1024,+ include: Optional[str] = None,+ exclude: Optional[str] = None,) -> Tuple[int, List[str]]:"""Upload an entire directory recursively and return uploaded count with warnings.Optimized: collects all files in one pass, pre-creates directories upfront,then uploads all files concurrently (up to _UPLOAD_CONCURRENCY at a time)."""- effective_ignore_dirs = ignore_dirs if ignore_dirs is not None else IGNORE_DIRSeffective_ignore_extensions = (ignore_extensions if ignore_extensions is not None else IGNORE_EXTENSIONS)gitignore_matcher = GitignoreMatcher(local_dir)+ # Keep repository uploads aligned with DirectoryParser's public filtering+ # contract without duplicating its path/glob semantics.+ from openviking.parse.directory_scan import (+ _matches_exclude,+ _matches_include,+ _parse_patterns,+ _should_skip_directory,+ )++ if isinstance(ignore_dirs, str):+ parsed_ignore_dirs: Optional[Set[str]] = set(_parse_patterns(ignore_dirs)) or None+ elif ignore_dirs:+ parsed_ignore_dirs = {str(item) for item in ignore_dirs}+ else:+ parsed_ignore_dirs = None+ include_patterns = _parse_patterns(include)+ exclude_patterns = _parse_patterns(exclude)warnings: List[str] = []@@ -171,7 +189,11 @@ async def upload_directory(kept = []for d in dirs:sub_dir_path = dir_path / d- should_skip = should_skip_directory(d, ignore_dirs=effective_ignore_dirs)+ should_skip, _ = _should_skip_directory(+ sub_dir_path,+ local_dir,+ parsed_ignore_dirs,+ )if should_skip:continue@@ -196,6 +218,14 @@ async def upload_directory(continuerel_path_str = str(file_path.relative_to(local_dir)).replace(os.sep, "/")+ if include_patterns and not _matches_include(file_name, include_patterns):+ continue+ if exclude_patterns and _matches_exclude(+ rel_path_str,+ file_name,+ exclude_patterns,+ ):+ continuetry:target_uri = safe_join_viking_uri(viking_uri_base, rel_path_str)except ValueError as exc:diff --git a/tests/parse/test_add_directory.py b/tests/parse/test_add_directory.pyindex d5c7a3f1e..1b3f687e4 100644--- a/tests/parse/test_add_directory.py+++ b/tests/parse/test_add_directory.py@@ -208,6 +208,37 @@ class TestDirectoryParserBasic:p = DirectoryParser()assert p.can_parse(tmp_path) is True+ @pytest.mark.asyncio+ async def test_git_repository_preserves_directory_filters(+ self, tmp_path: Path, fake_fs: FakeVikingFS+ ) -> None:+ (tmp_path / ".git").mkdir()+ (tmp_path / "notes").mkdir()+ (tmp_path / "08_Attachments").mkdir()+ (tmp_path / "notes" / "article.md").write_text("keep", encoding="utf-8")+ (tmp_path / "notes" / "private.excalidraw.md").write_text(+ "exclude", encoding="utf-8"+ )+ (tmp_path / "08_Attachments" / "diagram.md").write_text(+ "ignore", encoding="utf-8"+ )+ (tmp_path / "main.py").write_text("exclude by include", encoding="utf-8")++ with (+ patch.object(BaseParser, "_get_viking_fs", return_value=fake_fs),+ patch.object(DirectoryParser, "_add_git_metadata", new=AsyncMock()),+ ):+ result = await DirectoryParser().parse(+ str(tmp_path),+ ignore_dirs="08_Attachments",+ include="*.md",+ exclude="*.excalidraw.md",+ )++ uploaded_paths = {uri.split("/repository/", 1)[-1] for uri in fake_fs.files}+ assert result.parser_name == "CodeRepositoryParser"+ assert uploaded_paths == {"notes/article.md"}+def test_can_parse_file(self, tmp_path: Path):f = tmp_path / "test.md"f.write_text("hello")