Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(ingestion): keep hand-written files that mention codegen, and tracked files under ignored paths (#2689)

Swati Ahuja29d996aba3e3
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/command.py b/packages/cli/src/repowise/cli/commands/init_cmd/command.py
index 2240353f0..70a062166 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/command.py
@@ -1820,7 +1820,7 @@ def init_command(
base_state["git_tier"] = git_tier_for_run_mode(run_mode)
apply_git_history_coverage_state(base_state, result)
from repowise.core.generation.selection import count_documentable_files
- from repowise.core.index_scope import file_page_scope, stamp_index_scope
+ from repowise.core.index_scope import dropped_files_scope, file_page_scope, stamp_index_scope
_scope_embedder = embedder_name_resolved if not effective_index_only else _index_only_embedder
_unavailable = []
@@ -1836,6 +1836,7 @@ def init_command(
],
git_tier=git_tier_for_run_mode(run_mode),
git_commit_cap=resolved_commit_limit,
+ dropped_files=dropped_files_scope(getattr(result, "traversal_stats", None)),
file_pages={
"configured_cap": max_file_pages,
**(
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
index c0c4d65c1..ff7b61b31 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
@@ -29,7 +29,11 @@ from repowise.cli.state_persistence import build_kg_state, save_knowledge_graph_
from repowise.core.analysis.health import HEALTH_ANALYZER_VERSION
from repowise.core.docs_mode import docs_mode_state_fields
from repowise.core.generation.models import count_stub_fallbacks
-from repowise.core.index_scope import file_page_scope, stamp_index_scope
+from repowise.core.index_scope import (
+ dropped_files_scope,
+ file_page_scope,
+ stamp_index_scope,
+)
from repowise.core.repo_config import config_dependency_fingerprints
logger = structlog.get_logger(__name__)
@@ -370,6 +374,7 @@ def _stamp_full_init_scope(
content_provenance="model",
git_tier="full",
git_commit_cap=resolved_commit_limit,
+ dropped_files=dropped_files_scope(getattr(result, "traversal_stats", None)),
file_pages={"configured_cap": max_file_pages, **pages},
analysis={"unavailable": unavailable, "skipped": []},
provider={
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py b/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py
index 381eb050f..98f8d5883 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/workspace.py
@@ -504,7 +504,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCt
state["phase_timings"] = repo_phase_timings
apply_git_history_coverage_state(state, result)
from repowise.core.generation.selection import count_documentable_files
- from repowise.core.index_scope import file_page_scope, stamp_index_scope
+ from repowise.core.index_scope import dropped_files_scope, file_page_scope, stamp_index_scope
scope_config = load_config(repo.path)
configured_cap = resolve_max_file_pages(config=scope_config)
@@ -525,6 +525,7 @@ def _ingest_and_generate_repo(repo: Any, idx: int, total: int, ctx: _WorkspaceCt
content_provenance={"none": "none", "deterministic": "template", "llm": "model"}[docs_mode],
git_tier=state["git_tier"],
git_commit_cap=ctx.resolved_commit_limit,
+ dropped_files=dropped_files_scope(getattr(result, "traversal_stats", None)),
file_pages={
"configured_cap": configured_cap,
**(
diff --git a/packages/core/src/repowise/core/index_scope.py b/packages/core/src/repowise/core/index_scope.py
index 4b300529c..282ccc2ff 100644
--- a/packages/core/src/repowise/core/index_scope.py
+++ b/packages/core/src/repowise/core/index_scope.py
@@ -81,6 +81,45 @@ def _string(value: Any) -> str | None:
return value if isinstance(value, str) and value else None
+#: Dropped paths named per reason; the count stays exact. The scope is embedded
+#: in editor files and orientation responses, so it carries a sample, not a
+#: listing of every generated file in a codegen-heavy repo.
+_DROPPED_PATHS_SHOWN = 20
+
+
+def dropped_files_scope(stats: Any) -> dict[str, Any] | None:
+ """What traversal dropped as generated, from its ``TraversalStats``.
+
+ ``None`` when there are no stats, so a run that never walked the tree does
+ not claim it dropped nothing.
+ """
+ paths = getattr(stats, "generated_files", None)
+ if paths is None:
+ return None
+ ordered = sorted(paths)
+ return {
+ "generated": {
+ "count": len(ordered),
+ "paths": ordered[:_DROPPED_PATHS_SHOWN],
+ "truncated": len(ordered) > _DROPPED_PATHS_SHOWN,
+ }
+ }
+
+
+def _dropped_files(value: Any) -> dict[str, Any] | None:
+ if not isinstance(value, Mapping) or not isinstance(value.get("generated"), Mapping):
+ return None
+ generated = value["generated"]
+ paths = generated.get("paths")
+ return {
+ "generated": {
+ "count": _number(generated.get("count")),
+ "paths": [str(p) for p in paths] if isinstance(paths, list) else [],
+ "truncated": generated.get("truncated") is True,
+ }
+ }
+
+
def resolve_index_scope(
state: Mapping[str, Any] | None,
config: Mapping[str, Any] | None = None,
@@ -135,6 +174,7 @@ def resolve_index_scope(
if isinstance(scope.get("git_history_coverage"), Mapping)
else None
),
+ "dropped_files": _dropped_files(scope.get("dropped_files")),
"file_pages": {
"configured_cap": configured_cap,
"effective_cap": _number(pages.get("effective_cap")),
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index a02a4e3d2..9c31d8860 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -75,6 +75,25 @@ class SkippedSourceFile(NamedTuple):
reason: str # "over_max_size" | "minified"
+class _DirIgnore(NamedTuple):
+ """One directory's nested ignore rules, matched against a child's name.
+
+ Kept apart because they differ in authority: git never ignores a tracked
+ file, so a ``.gitignore`` rule yields to the tracked set while a
+ ``.repowiseIgnore`` rule is the user's own exclusion and always applies.
+ """
+
+ gitignore: pathspec.PathSpec
+ extra: pathspec.PathSpec
+
+
+class _TrackedPaths(NamedTuple):
+ """What git tracks under the root: files, and every directory holding one."""
+
+ files: frozenset[str]
+ dirs: frozenset[str]
+
+
@dataclass
class TraversalStats:
"""Counts collected during file traversal, broken down by skip reason."""
@@ -127,6 +146,13 @@ class TraversalStats:
"""
unknown_language_files_truncated: bool = False
"""True once the cap above dropped a name, so a reader can say "at least"."""
+ generated_files: list[str] = field(default_factory=list)
+ """Paths behind ``skipped_generated``, so the index scope can name them.
+
+ Uncapped: one short string per dropped file, bounded by the tree the
+ traverser already holds a record of, and a capped list filled by parallel
+ workers would name a different subset on every run.
+ """
log = structlog.get_logger(__name__)
@@ -249,15 +275,23 @@ _BLOCKED_FILENAME_SPEC: pathspec.PathSpec = pathspec.PathSpec.from_lines(
"gitwildmatch", _BLOCKED_FILENAME_PATTERNS
)
-# Generated file markers (checked in first 512 bytes)
-_GENERATED_MARKERS: tuple[str, ...] = (
- "Code generated",
- "DO NOT EDIT",
- "This file was automatically generated",
+# Generated-file banners, read only from comment lines and docstring openers
+# in the first two lines. These two count anywhere in the comment text:
+_GENERATED_ANYWHERE_MARKERS: tuple[str, ...] = ("DO NOT EDIT", "@GENERATED")
+# ...these only where the comment text starts ("<auto-generated>" included):
+_GENERATED_LEADING_MARKERS: tuple[str, ...] = (
+ "CODE GENERATED",
+ "THIS FILE WAS AUTOMATICALLY GENERATED",
"GENERATED CODE",
"AUTO-GENERATED",
- "@generated",
)
+# ...plus the sentence that says it anywhere in the comment.
+_THIS_FILE_GENERATED_RE = re.compile(
+ r"\bTHIS FILE (?:IS|WAS) (?:AUTO-?|AUTOMATICALLY )?GENERATED\b"
+)
+# Longest first, so "/*" is stripped whole rather than as "/".
+_COMMENT_PREFIXES: tuple[str, ...] = ("<!--", "//", "/*", "--", "#", "*", ";")
+_DOCSTRING_OPENER_RE = re.compile(r"""^[rRuUbB]?(?:\"\"\"|''')""")
_GENERATED_SUFFIXES: tuple[str, ...] = tuple(_LANG_REGISTRY.generated_suffixes())
@@ -353,9 +387,16 @@ class FileTraverser:
self._extra_exclude = _compile_gitignore(patterns)
# Absolute dir path -> that directory's nested ignore spec. The root is
# pre-seeded because its files are already loaded above.
- self._dir_ignore_cache: dict[str, pathspec.PathSpec] = {
- str(self.repo_root): self._extra_ignore,
+ self._dir_ignore_cache: dict[str, _DirIgnore] = {
+ str(self.repo_root): _DirIgnore(_compile_gitignore([]), self._extra_ignore),
}
+ # Lazy: read only once a .gitignore rule matches something.
+ self._tracked: _TrackedPaths | None = None
+ self._tracked_lock = threading.Lock()
+ # Gitignored dirs walked only for the tracked files they hold. A nested
+ # rule is matched against the child name alone, so without this the
+ # untracked files beside a tracked `.gitkeep` would slip in.
+ self._ignored_tracked_dirs: set[str] = set()
# Parsed even when submodules are included: the set exempts them from
# the nested-repo skip, since an initialized submodule has a `.git` file.
self._submodule_paths: frozenset[str] = _parse_gitmodules(self.repo_root)
@@ -490,12 +531,12 @@ class FileTraverser:
self.stats.total_paths_walked += 1
yield dirpath_obj / filename
- def _get_dir_ignore(self, dirpath: Path) -> pathspec.PathSpec:
- """Return the per-directory ignore spec, loading and caching on first access.
+ def _get_dir_ignore(self, dirpath: Path) -> _DirIgnore:
+ """Return the per-directory ignore specs, loading and caching on first access.
- Merges the directory's nested ``.gitignore`` and ``.repowiseIgnore``
- (in that order), as git applies a ``.gitignore`` to its own directory.
- Patterns are matched against the immediate child name.
+ The directory's nested ``.gitignore`` and ``.repowiseIgnore``, as git
+ applies a ``.gitignore`` to its own directory. Patterns are matched
+ against the immediate child name.
Read outside the lock and written under it, like
:meth:`_console_script_tables`, since the callers are per-path workers.
@@ -503,24 +544,48 @@ class FileTraverser:
key = str(dirpath)
spec = self._dir_ignore_cache.get(key)
if spec is None:
- lines: list[str] = []
- for name in (".gitignore", self._extra_ignore_filename):
- ignore_file = dirpath / name
- if ignore_file.exists():
- lines.extend(
- ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines()
- )
- spec = _compile_gitignore(lines)
+ spec = _DirIgnore(
+ load_extra_ignore_spec(dirpath, ".gitignore"),
+ load_extra_ignore_spec(dirpath, self._extra_ignore_filename),
+ )
with self._dir_ignore_lock:
# Keep the first published spec so every caller shares one object.
spec = self._dir_ignore_cache.setdefault(key, spec)
return spec
+ def _tracked_paths(self) -> _TrackedPaths:
+ """Git's tracked files under the root, read once; empty outside a repo.
+
+ Double-checked locking as in :meth:`_console_script_tables`: the first
+ caller may be a per-path worker, and each duplicate is a git call.
+ """
+ if self._tracked is None:
+ with self._tracked_lock:
+ if self._tracked is None:
+ # Deferred: git_refs reaches analysis modules that import ingestion.
+ from ..git_refs import tracked_paths
+
+ files = tracked_paths(str(self.repo_root))
+ dirs = {p.as_posix() for f in files for p in Path(f).parents}
+ dirs.discard(".")
+ self._tracked = _TrackedPaths(files, frozenset(dirs))
+ return self._tracked
+
+ def _is_tracked(self, rel_str: str, *, is_dir: bool = False) -> bool:
+ """Whether git tracks the file, or for a directory any file under it.
+
+ Git ignores only untracked paths, so a ``.gitignore`` match yields to
+ this: a broad rule such as ``logs`` cannot drop a file a nested
+ ``.gitignore`` re-included with ``!`` and git tracks.
+ """
+ tracked = self._tracked_paths()
+ return rel_str in (tracked.dirs if is_dir else tracked.files)
+
def _should_skip_dir(
self,
dirname: str,
rel_path: Path,
- dir_ignore: pathspec.PathSpec | None = None,
+ dir_ignore: _DirIgnore | None = None,
) -> bool:
if dirname in _BLOCKED_DIRS:
return True
@@ -528,14 +593,34 @@ class FileTraverser:
if self._is_repo_boundary(rel_str, self.repo_root / rel_path):
return True
dir_pattern = rel_str + "/"
- if (
- self._gitignore.match_file(dir_pattern)
- or self._extra_ignore.match_file(dir_pattern)
- or self._extra_exclude.match_file(dir_pattern)
+ if self._extra_ignore.match_file(dir_pattern) or self._extra_exclude.match_file(
+ dir_pattern
):
return True
# Per-directory ignore: pattern is relative to the parent directory.
- return dir_ignore is not None and dir_ignore.match_file(dirname + "/")
+ if dir_ignore is not None and dir_ignore.extra.match_file(dirname + "/"):
+ return True
+ nested = dir_ignore is not None and dir_ignore.gitignore.match_file(dirname + "/")
+ return self._dir_gitignored(rel_str, nested)
+
+ def _dir_gitignored(self, rel_str: str, nested_match: bool) -> bool:
+ """Whether ``.gitignore`` prunes the directory, which it may not if git tracks a file in it."""
+ if not (
+ nested_match
+ or self._gitignore.match_file(rel_str + "/")
+ or self._in_ignored_tracked_dir(rel_str)
+ ):
+ return False
+ if not self._is_tracked(rel_str, is_dir=True):
+ return True
+ self._ignored_tracked_dirs.add(rel_str)
+ return False
+
+ def _in_ignored_tracked_dir(self, rel_str: str) -> bool:
+ """Whether a gitignored directory kept for its tracked files holds *rel_str*."""
+ return bool(self._ignored_tracked_dirs) and any(
+ p.as_posix() in self._ignored_tracked_dirs for p in Path(rel_str).parents
+ )
def _is_repo_boundary(self, rel_str: str, abs_path: Path) -> bool:
"""True (and counted) for an excluded submodule or a nested git repo."""
@@ -607,19 +692,31 @@ class FileTraverser:
"""The stats counter of the first path rule that excludes this file, if any."""
if abs_path.suffix.lower() in _BLOCKED_EXTENSIONS:
return "skipped_blocked_extension"
- if self._gitignore.match_file(rel_str):
+ if self._root_gitignored(rel_str):
return "skipped_gitignore"
if self._extra_ignore.match_file(rel_str):
return "skipped_extra_ignore"
if self._extra_exclude.match_file(rel_str):
return "skipped_extra_exclude"
- # Per-directory .repowiseIgnore: check filename against the parent dir's spec.
- if self._get_dir_ignore(abs_path.parent).match_file(abs_path.name):
+ if self._dir_ignore_drops(abs_path, rel_str):
return "skipped_dir_ignore"
if self._blocked_patterns.match_file(rel_str):
return "skipped_blocked_pattern"
return None
+ def _root_gitignored(self, rel_str: str) -> bool:
+ """Whether the root ``.gitignore`` (or a kept ignored dir) drops an untracked file."""
+ return (
+ self._gitignore.match_file(rel_str) or self._in_ignored_tracked_dir(rel_str)
+ ) and not self._is_tracked(rel_str)
+
+ def _dir_ignore_drops(self, abs_path: Path, rel_str: str) -> bool:
+ """Whether the parent directory's own ignore files drop the file, by name."""
+ dir_ignore = self._get_dir_ignore(abs_path.parent)
+ if dir_ignore.extra.match_file(abs_path.name):
+ return True
+ return dir_ignore.gitignore.match_file(abs_path.name) and not self._is_tracked(rel_str)
+
def _resolve_language(
self, abs_path: Path, rel_str: str, size_bytes: int
) -> LanguageTag | None:
@@ -664,7 +761,9 @@ class FileTraverser:
or not _is_generated(abs_path)
):
return False
- self._count("skipped_generated")
+ with self._count_lock:
+ self.stats.skipped_generated += 1
+ self.stats.generated_files.append(rel_str)
log.debug("Skipping generated file", path=rel_str)
return True
@@ -930,8 +1029,11 @@ def _looks_minified(abs_path: Path) -> bool | None:
def _is_generated(abs_path: Path) -> bool:
"""Return True if the file appears to be auto-generated.
- A generated-file banner sits on the first line or two. Requiring the
- marker there keeps the check off docblocks that merely mention one.
+ A generated-file banner sits on the first line or two, in a comment or a
+ module docstring. Code and prose that merely mention codegen (a string
+ literal "Auto-generated by …", a docstring about "the auto-generated
+ client") are not banners, so a marker counts only as :func:`_is_banner`
+ reads it.
"""
name = abs_path.name
if any(name.endswith(sfx) for sfx in _GENERATED_SUFFIXES):
@@ -941,8 +1043,39 @@ def _is_generated(abs_path: Path) -> bool:
header = f.read(512)
except OSError:
return False
- banner = "\n".join(header.splitlines()[:2]).upper()
- return any(marker.upper() in banner for marker in _GENERATED_MARKERS)
+ in_docstring = False
+ for line in header.splitlines()[:2]:
+ text = _comment_text(line.strip(), in_docstring)
+ if text is None:
+ continue
+ # A bare opener ('"""' alone) puts the docstring text on the next line.
+ in_docstring = not text and bool(_DOCSTRING_OPENER_RE.match(line.strip()))
+ if _is_banner(text.upper()):
+ return True
+ return False
+
+
+def _comment_text(line: str, in_docstring: bool) -> str | None:
+ """The text of a comment line or docstring opener, None for anything else."""
+ if in_docstring:
+ return line
+ if opener := _DOCSTRING_OPENER_RE.match(line):
+ return line[opener.end() :].strip()
+ for prefix in _COMMENT_PREFIXES:
+ if line.startswith(prefix):
+ return line[len(prefix) :].strip()
+ return None
+
+
+def _is_banner(text: str) -> bool:
+ """Whether upper-cased comment *text* declares its file generated."""
+ if any(marker in text for marker in _GENERATED_ANYWHERE_MARKERS):
+ return True
+ if _THIS_FILE_GENERATED_RE.search(text):
+ return True
+ # Leading punctuation is decoration ("<auto-generated>", "*** GENERATED").
+ lead = text.lstrip("<!*-=#/; \t")
+ return lead.startswith(_GENERATED_LEADING_MARKERS)
def _is_config_file(language: LanguageTag) -> bool:
diff --git a/tests/unit/core/test_index_scope.py b/tests/unit/core/test_index_scope.py
index 39044efb9..dbb19fda1 100644
--- a/tests/unit/core/test_index_scope.py
+++ b/tests/unit/core/test_index_scope.py
@@ -56,3 +56,23 @@ def test_mixed_provenance_survives_resolution() -> None:
state = {"run_mode": "standard", "docs_mode": "llm"}
stamp_index_scope(state, {}, content_provenance="mixed")
assert resolve_index_scope(state, {})["content_provenance"] == "mixed"
+
+
+def test_dropped_generated_files_are_named_in_the_scope() -> None:
+ from repowise.core.index_scope import dropped_files_scope
+
+ stats = SimpleNamespace(generated_files=[f"gen/f{i:02}.go" for i in range(25, -1, -1)])
+ state: dict = {}
+ stamp_index_scope(state, dropped_files=dropped_files_scope(stats))
+ dropped = resolve_index_scope(state)["dropped_files"]["generated"]
+ assert dropped["count"] == 26
+ assert dropped["paths"][:2] == ["gen/f00.go", "gen/f01.go"]
+ assert len(dropped["paths"]) == 20
+ assert dropped["truncated"] is True
+
+
+def test_scope_without_traversal_does_not_claim_nothing_dropped() -> None:
+ from repowise.core.index_scope import dropped_files_scope
+
+ assert dropped_files_scope(None) is None
+ assert resolve_index_scope({})["dropped_files"] is None
diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.py
index 48621307c..c0aa43ad8 100644
--- a/tests/unit/ingestion/test_traverser.py
+++ b/tests/unit/ingestion/test_traverser.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import subprocess
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
@@ -571,6 +572,84 @@ class TestNestedGitignore:
assert not any("comp.snap" in p for p in paths)
+class TestTrackedFilesNeverGitignored:
+ """Git never ignores a tracked file, so a ``.gitignore`` rule here can't either.
+
+ The shape that dropped a real doc: a root ``logs`` rule, and a nested
+ ``.gitignore`` re-including one file with ``!`` that git then tracks.
+ """
+
+ @staticmethod
+ def _git_add(root: Path, *paths: str) -> None:
+ subprocess.run(["git", "init", "-q"], cwd=root, check=True)
+ subprocess.run(["git", "add", "-f", *paths], cwd=root, check=True)
+
+ def _repo(self, root: Path) -> None:
+ (root / ".gitignore").write_text("logs\n")
+ docs = root / "docs"
+ (docs / "logs").mkdir(parents=True)
+ (docs / ".gitignore").write_text("!logs/\n!logs/overview.py\n")
+ (docs / "logs" / "overview.py").write_text("x = 1\n")
+ (docs / "logs" / "scratch.py").write_text("y = 2\n")
+ (root / "app.py").write_text("pass\n")
+
+ def test_tracked_file_under_ignored_dir_indexed(self, tmp_path: Path) -> None:
+ self._repo(tmp_path)
+ self._git_add(tmp_path, "app.py", "docs/logs/overview.py")
+ tv = FileTraverser(tmp_path)
+ paths = {f.path for f in tv.traverse()}
+ assert "docs/logs/overview.py" in paths
+ assert not tv.dir_chain_skipped(Path("docs/logs"))
+
+ def test_untracked_ignored_still_skipped(self, tmp_path: Path) -> None:
+ self._repo(tmp_path)
+ self._git_add(tmp_path, "app.py", "docs/logs/overview.py")
+ (tmp_path / "logs").mkdir()
+ (tmp_path / "logs" / "run.py").write_text("pass\n")
+ paths = {f.path for f in FileTraverser(tmp_path).traverse()}
+ assert "docs/logs/scratch.py" not in paths
+ assert "logs/run.py" not in paths
+ assert "app.py" in paths
+
+ def test_nested_gitignore_yields_to_tracked_file(self, tmp_path: Path) -> None:
+ (tmp_path / "pkg").mkdir()
+ (tmp_path / "pkg" / ".gitignore").write_text("gen_*.py\n")
+ (tmp_path / "pkg" / "gen_kept.py").write_text("pass\n")
+ (tmp_path / "pkg" / "gen_scratch.py").write_text("pass\n")
+ self._git_add(tmp_path, "pkg/gen_kept.py")
+ paths = {f.path for f in FileTraverser(tmp_path).traverse()}
+ assert "pkg/gen_kept.py" in paths
+ assert "pkg/gen_scratch.py" not in paths
+
+ def test_untracked_beside_tracked_keep_file_in_nested_ignored_dir_skipped(
+ self, tmp_path: Path
+ ) -> None:
+ web = tmp_path / "web"
+ (web / "gen" / "assets").mkdir(parents=True)
+ (web / ".gitignore").write_text("gen/\n")
+ (web / "gen" / "keep.py").write_text("pass\n")
+ (web / "gen" / "bundle.py").write_text("pass\n")
+ (web / "gen" / "assets" / "chunk.py").write_text("pass\n")
+ self._git_add(tmp_path, "web/gen/keep.py")
+ paths = {f.path for f in FileTraverser(tmp_path).traverse()}
+ assert paths == {"web/gen/keep.py"}
+
+ def test_repowise_ignore_and_excludes_still_drop_tracked_files(self, tmp_path: Path) -> None:
+ self._repo(tmp_path)
+ (tmp_path / "vendor").mkdir()
+ (tmp_path / "vendor" / "lib.py").write_text("pass\n")
+ (tmp_path / ".repowiseIgnore").write_text("docs/\n")
+ self._git_add(tmp_path, "app.py", "docs/logs/overview.py", "vendor/lib.py")
+ tv = FileTraverser(tmp_path, extra_exclude_patterns=["vendor/"])
+ assert {f.path for f in tv.traverse()} == {"app.py"}
+
+ def test_non_git_checkout_keeps_gitignore_rules(self, tmp_path: Path) -> None:
+ self._repo(tmp_path)
+ paths = {f.path for f in FileTraverser(tmp_path).traverse()}
+ assert "docs/logs/overview.py" not in paths
+ assert "app.py" in paths
+
+
# ---------------------------------------------------------------------------
# Monorepo detection
# ---------------------------------------------------------------------------
@@ -1175,6 +1254,64 @@ class TestIsGenerated:
p.write_text("x = 1\n")
assert _is_generated(p) is True
+ def test_generated_marker_in_string_literal_kept(self, tmp_path: Path) -> None:
+ # The constant a code generator stamps on its output is not a banner.
+ p = tmp_path / "constants.ts"
+ p.write_text("export const BANNER = 'Auto-generated by the CLI. Do not modify.';\n")
+ assert _is_generated(p) is False
+
+ def test_docstring_mentions_autogenerated_kept(self, tmp_path: Path) -> None:
+ p = tmp_path / "__init__.py"
+ p.write_text('"""\nThis module is a light wrapper around the auto-generated client.\n"""\n')
+ assert _is_generated(p) is False
+
+ def test_go_code_generated_header_dropped(self, tmp_path: Path) -> None:
+ p = tmp_path / "api.pb.gw.go"
+ p.write_text(
+ "// Code generated by protoc-gen-grpc-gateway. DO NOT EDIT.\n"
+ "// source: api.proto\n\npackage api\n"
+ )
+ assert _is_generated(p) is True
+
+ @pytest.mark.parametrize(
+ "header",
+ [
+ "# Generated by the protocol buffer compiler. DO NOT EDIT!\n",
+ "/** @generated */\n",
+ "// <auto-generated/>\n",
+ '"""Auto-generated by schema_gen.py."""\n',
+ '"""\nAuto-generated by schema_gen.py.\n"""\n',
+ "# NOTE: this file is auto-generated by gen.py, edit the template\n",
+ "#!/usr/bin/env node\n// AUTO-GENERATED by setup\n",
+ ],
+ )
+ def test_real_banners_still_dropped(self, tmp_path: Path, header: str) -> None:
+ p = tmp_path / "gen.py"
+ p.write_text(header + "x = 1\n")
+ assert _is_generated(p) is True
+
+ @pytest.mark.parametrize(
+ "header",
+ [
+ "# Helpers for code generated by the compiler plugin.\n",
+ '"""Utilities for reading GENERATED CODE maps."""\n',
+ 'MSG = "Code generated by X. DO NOT EDIT."\n',
+ ],
+ )
+ def test_marker_not_leading_the_comment_kept(self, tmp_path: Path, header: str) -> None:
+ p = tmp_path / "mod.py"
+ p.write_text(header + "x = 1\n")
+ assert _is_generated(p) is False
+
+ def test_dropped_generated_paths_recorded(self, tmp_path: Path) -> None:
+ (tmp_path / "gen.go").write_text("// Code generated by tool. DO NOT EDIT.\npackage x\n")
+ (tmp_path / "main.go").write_text("package x\n")
+ traverser = FileTraverser(tmp_path)
+ paths = [f.path for f in traverser.traverse()]
+ assert paths == ["main.go"]
+ assert traverser.stats.skipped_generated == 1
+ assert traverser.stats.generated_files == ["gen.go"]
+
class TestConcurrentLazyInit:
"""The lazy scans behind ``_build_file_info`` run once, not once per worker.