Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(ingestion): tolerate malformed .gitignore patterns (#1095)

ShotaFuti771dd107d338
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index e36779089..c74996372 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -18,13 +18,14 @@ from __future__ import annotations
import configparser
import os
import threading
-from collections.abc import Iterator
+from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
import pathspec
import structlog
+from pathspec.patterns.gitwildmatch import GitWildMatchPattern, GitWildMatchPatternError
from ..test_paths import is_test_related_path
from .languages.registry import REGISTRY as _LANG_REGISTRY
@@ -261,7 +262,7 @@ class FileTraverser:
"gitwildmatch", _BLOCKED_FILENAME_PATTERNS
)
patterns = extra_exclude_patterns or []
- self._extra_exclude = pathspec.PathSpec.from_lines("gitwildmatch", patterns)
+ self._extra_exclude = _compile_gitignore(patterns)
# Per-directory ignore cache: absolute dir path -> PathSpec built from
# that directory's nested .gitignore + .repowiseIgnore.
# Pre-seed root: its .gitignore is matched full-path via self._gitignore
@@ -392,7 +393,7 @@ class FileTraverser:
lines.extend(
ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines()
)
- self._dir_ignore_cache[key] = pathspec.PathSpec.from_lines("gitwildmatch", lines)
+ self._dir_ignore_cache[key] = _compile_gitignore(lines)
return self._dir_ignore_cache[key]
def _should_skip_dir(
@@ -811,6 +812,28 @@ def _parse_gitmodules(repo_root: Path) -> frozenset[str]:
return frozenset()
+def _compile_gitignore(lines: Iterable[str]) -> pathspec.PathSpec:
+ """Build a gitwildmatch ``PathSpec``, tolerating malformed patterns.
+
+ Git itself accepts ignore lines that ``pathspec`` rejects — e.g. a trailing
+ backslash like ``.godot\\`` (a real pattern found in Godot ``.gitignore``
+ files). ``PathSpec.from_lines`` raises ``GitWildMatchPatternError`` on the
+ first such line, which would abort ingestion of the entire repository over
+ one stray line that ``git status`` handles without complaint. Compile each
+ line independently and skip (with a warning) only the offending ones, so the
+ rest of the ignore file still applies.
+ """
+ patterns: list[GitWildMatchPattern] = []
+ for line in lines:
+ if not line:
+ continue
+ try:
+ patterns.append(GitWildMatchPattern(line))
+ except GitWildMatchPatternError:
+ log.warning("skipping malformed gitignore pattern", pattern=line)
+ return pathspec.PathSpec(patterns)
+
+
def _load_gitignore_spec(repo_root: Path) -> pathspec.PathSpec:
"""Root ignore spec: ``.gitignore`` merged with ``.git/info/exclude``.
@@ -826,7 +849,7 @@ def _load_gitignore_spec(repo_root: Path) -> pathspec.PathSpec:
):
if ignore_file.exists():
lines.extend(ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines())
- return pathspec.PathSpec.from_lines("gitwildmatch", lines)
+ return _compile_gitignore(lines)
def _load_extra_ignore_spec(repo_root: Path, filename: str) -> pathspec.PathSpec:
@@ -834,4 +857,4 @@ def _load_extra_ignore_spec(repo_root: Path, filename: str) -> pathspec.PathSpec
lines: list[str] = []
if ignore_file.exists():
lines = ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines()
- return pathspec.PathSpec.from_lines("gitwildmatch", lines)
+ return _compile_gitignore(lines)
diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.py
index 5c404ae34..dfc9779f2 100644
--- a/tests/unit/ingestion/test_traverser.py
+++ b/tests/unit/ingestion/test_traverser.py
@@ -7,7 +7,44 @@ from pathlib import Path
import pytest
from repowise.core.ingestion import traverser as traverser_mod
-from repowise.core.ingestion.traverser import FileTraverser, _detect_language
+from repowise.core.ingestion.traverser import (
+ FileTraverser,
+ _compile_gitignore,
+ _detect_language,
+)
+
+
+class TestCompileGitignore:
+ def test_malformed_line_is_not_fatal_and_keeps_valid(self) -> None:
+ spec = _compile_gitignore([".godot\\", "build/", "*.log", "", "# note"])
+ assert spec.match_file("build/x")
+ assert spec.match_file("a.log")
+ # A malformed line never takes down its neighbours.
+ assert not spec.match_file("src/main.py")
+
+ def test_trailing_backslash_matches_nothing(self) -> None:
+ # Git parity: git treats a dangling trailing backslash (e.g. `.godot\`)
+ # as an escape of nothing, so the pattern matches nothing — no error, no
+ # fallback to the bare path. Dropping the line reproduces that exactly.
+ # Do NOT re-add a salvage that rewrites `.godot\` -> `.godot`: that would
+ # diverge from git (it would start ignoring `.godot/` that git tracks).
+ spec = _compile_gitignore([".godot\\"])
+ assert not spec.match_file(".godot")
+ assert not spec.match_file(".godot/imported/x.res")
+ assert not spec.match_file("godot.py")
+
+ def test_unrecoverable_line_is_dropped(self) -> None:
+ # A lone backslash is unparseable and empty once stripped -> drop it,
+ # do not raise, and keep the valid neighbour.
+ spec = _compile_gitignore(["\\", "keep-me/"])
+ assert spec.match_file("keep-me/x")
+
+ def test_all_valid_lines_preserved(self) -> None:
+ spec = _compile_gitignore(["dist/", "*.tmp"])
+ assert spec.match_file("dist/a")
+ assert spec.match_file("x.tmp")
+ assert not spec.match_file("keep.py")
+
# ---------------------------------------------------------------------------
# Language detection
@@ -173,6 +210,21 @@ class TestFileTraverser:
assert not any("local-stash" in p for p in paths)
assert not any("notes.scratch" in p for p in paths)
+ def test_malformed_gitignore_line_does_not_abort(self, tmp_path: Path) -> None:
+ # Git tolerates patterns that pathspec rejects (e.g. a trailing
+ # backslash like ``.godot\``). One such line must not crash the whole
+ # traversal; the remaining valid patterns must still apply.
+ (tmp_path / ".gitignore").write_text(".godot\\\n*.log\napp_ok.py\n")
+ (tmp_path / "app.py").write_text("pass")
+ (tmp_path / "app_ok.py").write_text("pass")
+ (tmp_path / "debug.log").write_text("logs")
+ traverser = FileTraverser(tmp_path)
+ paths = [f.path for f in traverser.traverse()]
+ # Did not raise, and the well-formed patterns after the bad line held.
+ assert any("app.py" in p for p in paths)
+ assert not any("app_ok.py" in p for p in paths)
+ assert not any("debug.log" in p for p in paths)
+
def test_skips_oversized_files(self, tmp_path: Path) -> None:
big = tmp_path / "big.py"
big.write_bytes(b"x = 1\n" * 200_000) # ~1.2 MB