Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(ingestion): tolerate malformed .gitignore patterns (#1095)
ShotaFuti771dd107d338
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.pyindex e36779089..c74996372 100644--- a/packages/core/src/repowise/core/ingestion/traverser.py+++ b/packages/core/src/repowise/core/ingestion/traverser.py@@ -18,13 +18,14 @@ from __future__ import annotationsimport configparserimport osimport threading-from collections.abc import Iterator+from collections.abc import Iterable, Iteratorfrom dataclasses import dataclass, fieldfrom datetime import datetimefrom pathlib import Pathimport pathspecimport structlog+from pathspec.patterns.gitwildmatch import GitWildMatchPattern, GitWildMatchPatternErrorfrom ..test_paths import is_test_related_pathfrom .languages.registry import REGISTRY as _LANG_REGISTRY@@ -261,7 +262,7 @@ class FileTraverser:"gitwildmatch", _BLOCKED_FILENAME_PATTERNS)patterns = extra_exclude_patterns or []- self._extra_exclude = pathspec.PathSpec.from_lines("gitwildmatch", patterns)+ self._extra_exclude = _compile_gitignore(patterns)# Per-directory ignore cache: absolute dir path -> PathSpec built from# that directory's nested .gitignore + .repowiseIgnore.# Pre-seed root: its .gitignore is matched full-path via self._gitignore@@ -392,7 +393,7 @@ class FileTraverser:lines.extend(ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines())- self._dir_ignore_cache[key] = pathspec.PathSpec.from_lines("gitwildmatch", lines)+ self._dir_ignore_cache[key] = _compile_gitignore(lines)return self._dir_ignore_cache[key]def _should_skip_dir(@@ -811,6 +812,28 @@ def _parse_gitmodules(repo_root: Path) -> frozenset[str]:return frozenset()+def _compile_gitignore(lines: Iterable[str]) -> pathspec.PathSpec:+ """Build a gitwildmatch ``PathSpec``, tolerating malformed patterns.++ Git itself accepts ignore lines that ``pathspec`` rejects — e.g. a trailing+ backslash like ``.godot\\`` (a real pattern found in Godot ``.gitignore``+ files). ``PathSpec.from_lines`` raises ``GitWildMatchPatternError`` on the+ first such line, which would abort ingestion of the entire repository over+ one stray line that ``git status`` handles without complaint. Compile each+ line independently and skip (with a warning) only the offending ones, so the+ rest of the ignore file still applies.+ """+ patterns: list[GitWildMatchPattern] = []+ for line in lines:+ if not line:+ continue+ try:+ patterns.append(GitWildMatchPattern(line))+ except GitWildMatchPatternError:+ log.warning("skipping malformed gitignore pattern", pattern=line)+ return pathspec.PathSpec(patterns)++def _load_gitignore_spec(repo_root: Path) -> pathspec.PathSpec:"""Root ignore spec: ``.gitignore`` merged with ``.git/info/exclude``.@@ -826,7 +849,7 @@ def _load_gitignore_spec(repo_root: Path) -> pathspec.PathSpec:):if ignore_file.exists():lines.extend(ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines())- return pathspec.PathSpec.from_lines("gitwildmatch", lines)+ return _compile_gitignore(lines)def _load_extra_ignore_spec(repo_root: Path, filename: str) -> pathspec.PathSpec:@@ -834,4 +857,4 @@ def _load_extra_ignore_spec(repo_root: Path, filename: str) -> pathspec.PathSpeclines: list[str] = []if ignore_file.exists():lines = ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines()- return pathspec.PathSpec.from_lines("gitwildmatch", lines)+ return _compile_gitignore(lines)diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.pyindex 5c404ae34..dfc9779f2 100644--- a/tests/unit/ingestion/test_traverser.py+++ b/tests/unit/ingestion/test_traverser.py@@ -7,7 +7,44 @@ from pathlib import Pathimport pytestfrom repowise.core.ingestion import traverser as traverser_mod-from repowise.core.ingestion.traverser import FileTraverser, _detect_language+from repowise.core.ingestion.traverser import (+ FileTraverser,+ _compile_gitignore,+ _detect_language,+)+++class TestCompileGitignore:+ def test_malformed_line_is_not_fatal_and_keeps_valid(self) -> None:+ spec = _compile_gitignore([".godot\\", "build/", "*.log", "", "# note"])+ assert spec.match_file("build/x")+ assert spec.match_file("a.log")+ # A malformed line never takes down its neighbours.+ assert not spec.match_file("src/main.py")++ def test_trailing_backslash_matches_nothing(self) -> None:+ # Git parity: git treats a dangling trailing backslash (e.g. `.godot\`)+ # as an escape of nothing, so the pattern matches nothing — no error, no+ # fallback to the bare path. Dropping the line reproduces that exactly.+ # Do NOT re-add a salvage that rewrites `.godot\` -> `.godot`: that would+ # diverge from git (it would start ignoring `.godot/` that git tracks).+ spec = _compile_gitignore([".godot\\"])+ assert not spec.match_file(".godot")+ assert not spec.match_file(".godot/imported/x.res")+ assert not spec.match_file("godot.py")++ def test_unrecoverable_line_is_dropped(self) -> None:+ # A lone backslash is unparseable and empty once stripped -> drop it,+ # do not raise, and keep the valid neighbour.+ spec = _compile_gitignore(["\\", "keep-me/"])+ assert spec.match_file("keep-me/x")++ def test_all_valid_lines_preserved(self) -> None:+ spec = _compile_gitignore(["dist/", "*.tmp"])+ assert spec.match_file("dist/a")+ assert spec.match_file("x.tmp")+ assert not spec.match_file("keep.py")+# ---------------------------------------------------------------------------# Language detection@@ -173,6 +210,21 @@ class TestFileTraverser:assert not any("local-stash" in p for p in paths)assert not any("notes.scratch" in p for p in paths)+ def test_malformed_gitignore_line_does_not_abort(self, tmp_path: Path) -> None:+ # Git tolerates patterns that pathspec rejects (e.g. a trailing+ # backslash like ``.godot\``). One such line must not crash the whole+ # traversal; the remaining valid patterns must still apply.+ (tmp_path / ".gitignore").write_text(".godot\\\n*.log\napp_ok.py\n")+ (tmp_path / "app.py").write_text("pass")+ (tmp_path / "app_ok.py").write_text("pass")+ (tmp_path / "debug.log").write_text("logs")+ traverser = FileTraverser(tmp_path)+ paths = [f.path for f in traverser.traverse()]+ # Did not raise, and the well-formed patterns after the bad line held.+ assert any("app.py" in p for p in paths)+ assert not any("app_ok.py" in p for p in paths)+ assert not any("debug.log" in p for p in paths)+def test_skips_oversized_files(self, tmp_path: Path) -> None:big = tmp_path / "big.py"big.write_bytes(b"x = 1\n" * 200_000) # ~1.2 MB