Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix: honor include-submodules across incremental updates and upgrades (#381)
Raghav Chamadiya93764d68b1b7
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/command.py b/packages/cli/src/repowise/cli/commands/init_cmd/command.pyindex c1e902813..33d7a29ba 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/command.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/command.py@@ -806,6 +806,10 @@ def init_command(# same tier instead of silently upgrading ESSENTIAL → FULL (issue #341).base_state["run_mode"] = run_modebase_state["git_tier"] = git_tier_for_run_mode(run_mode)+ # Record whether submodules were indexed so `repowise update` rebuilds+ # the graph with the same boundary semantics (same pattern as git_tier:+ # missing → False keeps legacy behavior for old state files).+ base_state["include_submodules"] = include_submodulesif phase_timings:base_state["phase_timings"] = phase_timingskg = getattr(result, "knowledge_graph_result", None)@@ -835,6 +839,7 @@ def init_command(commit_limit=commit_limit,resolved_commit_limit=resolved_commit_limit,resolved_reasoning=resolved_reasoning,+ include_submodules=include_submodules,)# ---- Completion panel ----diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.pyindex 9862be6f6..8d9f4ec1a 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py@@ -177,6 +177,7 @@ def save_full_state_and_config(commit_limit: int | None,resolved_commit_limit: int,resolved_reasoning: str,+ include_submodules: bool = False,) -> None:"""Persist state.json + config for a completed full-mode (docs) init run."""@@ -216,6 +217,9 @@ def save_full_state_and_config(# Full-mode docs runs always index the FULL git tier.state["run_mode"] = "standard"state["git_tier"] = "full"+ # Same pattern as git_tier: `repowise update` reads this back so its+ # graph rebuild keeps the init run's submodule boundary semantics.+ state["include_submodules"] = include_submodulestotal_tokens = sum(p.total_tokens for p in (result.generated_pages or []))state["total_tokens"] = total_tokensif phase_timings:diff --git a/packages/cli/src/repowise/cli/commands/update_cmd.py b/packages/cli/src/repowise/cli/commands/update_cmd.pyindex 1dd532e8c..2e5423a2c 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd.py@@ -330,6 +330,8 @@ def _build_repo_graph(exclude_patterns: list[str],*,collect_sources: bool = False,+ include_submodules: bool = False,+ include_nested_repos: bool = False,) -> tuple[list, dict[str, bytes], Any, Any, int]:"""Traverse + parse the repo and build the graph (+ framework-aware edges).@@ -348,7 +350,12 @@ def _build_repo_graph(from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder, compute_content_hash- traverser = FileTraverser(repo_path, extra_exclude_patterns=exclude_patterns or None)+ traverser = FileTraverser(+ repo_path,+ extra_exclude_patterns=exclude_patterns or None,+ include_submodules=include_submodules,+ include_nested_repos=include_nested_repos,+ )file_infos = list(traverser.traverse())repo_structure = traverser.get_repo_structure()@@ -371,6 +378,8 @@ def _build_repo_graph(repo_path,exclude_patterns=exclude_patterns,centrality_cache_dir=PathlibPath(repo_path) / ".repowise",+ include_submodules=include_submodules,+ include_nested_repos=include_nested_repos,)skipped = 0@@ -419,6 +428,8 @@ def _rebuild_graph_and_git(cfg: dict,exclude_patterns: list[str],git_tier: str | None = None,+ include_submodules: bool = False,+ include_nested_repos: bool = False,) -> tuple[list, dict[str, bytes], Any, Any, int, dict[str, dict]]:"""Re-traverse + parse the repo, rebuild the graph (+ framework edges), andre-index git metadata for the changed files.@@ -428,12 +439,21 @@ def _rebuild_graph_and_git(its index never had. Unknown/missing values fall back to FULL, matchingthe historical behavior for legacy state files.+ ``include_submodules`` / ``include_nested_repos`` are likewise read from+ state.json: a repo indexed with ``init --include-submodules`` must not+ silently drop its submodule files on every incremental update. Missing+ keys fall back to False (legacy behavior).+Returns ``(parsed_files, source_map, graph_builder, repo_structure,file_count, git_meta_map)``."""# Full re-ingest for graph (needed for cascade analysis)parsed_files, source_map, graph_builder, repo_structure, file_count = _build_repo_graph(- repo_path, exclude_patterns, collect_sources=True+ repo_path,+ exclude_patterns,+ collect_sources=True,+ include_submodules=include_submodules,+ include_nested_repos=include_nested_repos,)# Re-index git metadata for changed files@@ -720,7 +740,10 @@ def _run_full_health_rescore(# Share the rebuild path with the incremental update so both produce the# same graph (same parser, same framework-aware synthetic edges).parsed_files, _source_map, graph_builder, _repo_structure, _file_count = _build_repo_graph(- repo_path, exclude_patterns+ repo_path,+ exclude_patterns,+ include_submodules=bool(state.get("include_submodules", False)),+ include_nested_repos=bool(state.get("include_nested_repos", False)),)# Fan-out metric precompute (mirrors _rebuild_graph_and_git) — the@@ -1145,7 +1168,13 @@ def update_command(parsed_files, source_map, graph_builder, repo_structure, file_count, git_meta_map = (_rebuild_graph_and_git(- repo_path, file_diffs, cfg, exclude_patterns, git_tier=state.get("git_tier")+ repo_path,+ file_diffs,+ cfg,+ exclude_patterns,+ git_tier=state.get("git_tier"),+ include_submodules=bool(state.get("include_submodules", False)),+ include_nested_repos=bool(state.get("include_nested_repos", False)),))diff --git a/packages/cli/src/repowise/cli/commands/upgrade_flow.py b/packages/cli/src/repowise/cli/commands/upgrade_flow.pyindex bf5c7c88a..5ed3453c1 100644--- a/packages/cli/src/repowise/cli/commands/upgrade_flow.py+++ b/packages/cli/src/repowise/cli/commands/upgrade_flow.py@@ -52,7 +52,16 @@ def _reparse(repo_path: Path, exclude_patterns: list[str]) -> tuple[list[Any], d"""from repowise.core.ingestion import ASTParser, FileTraverser- traverser = FileTraverser(repo_path, extra_exclude_patterns=exclude_patterns or None)+ # Honor the persisted submodule semantics of the original index — a+ # fast index built with --include-submodules must not drop submodule+ # files from the docs re-parse (missing key → False, legacy behavior).+ state = load_state(repo_path)+ traverser = FileTraverser(+ repo_path,+ extra_exclude_patterns=exclude_patterns or None,+ include_submodules=bool(state.get("include_submodules", False)),+ include_nested_repos=bool(state.get("include_nested_repos", False)),+ )file_infos = list(traverser.traverse())repo_structure = traverser.get_repo_structure()diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.pyindex b16de7a4a..e28de904c 100644--- a/packages/core/src/repowise/core/ingestion/traverser.py+++ b/packages/core/src/repowise/core/ingestion/traverser.py@@ -227,9 +227,11 @@ class FileTraverser:self._dir_ignore_cache: dict[str, pathspec.PathSpec] = {str(self.repo_root): self._extra_ignore,}- self._submodule_paths: frozenset[str] = frozenset()- if not include_submodules:- self._submodule_paths = _parse_gitmodules(self.repo_root)+ # Parse .gitmodules unconditionally: when submodules are *included*+ # the set is what exempts initialized submodules (whose `.git` file+ # makes them look like nested repos) from the nested-git skip below.+ self._submodule_paths: frozenset[str] = _parse_gitmodules(self.repo_root)+ self._include_submodules = include_submodulesself._include_nested_repos = include_nested_reposself.stats = TraversalStats()self._count_lock = threading.Lock()@@ -238,7 +240,7 @@ class FileTraverser:repo_root=str(self.repo_root),max_file_size_kb=max_file_size_kb,extra_exclude_patterns=len(patterns),- submodules_skipped=len(self._submodule_paths),+ submodules_skipped=0 if include_submodules else len(self._submodule_paths),include_nested_repos=include_nested_repos,)@@ -352,15 +354,19 @@ class FileTraverser:if dirname in _BLOCKED_DIRS:return Truerel_str = rel_path.as_posix()- if rel_str in self._submodule_paths:+ is_submodule = rel_str in self._submodule_paths+ if is_submodule and not self._include_submodules:self.stats.skipped_submodule += 1return True# Nested git repos are independent units — stop at the boundary# unless the caller explicitly opted in. Mirrors the workspace# scanner, which already refuses to descend into nested `.git`# markers. Without this, a parent repo that physically contains- # sibling repos gets walked end-to-end.- if not self._include_nested_repos and _is_nested_git_repo(abs_path):+ # sibling repos gets walked end-to-end. An *initialized* submodule+ # carries a `.git` file and would match here too — submodules that+ # were explicitly opted in above are exempt (they still fall through+ # to the gitignore/exclude checks below).+ if not self._include_nested_repos and not is_submodule and _is_nested_git_repo(abs_path):self.stats.skipped_nested_repo += 1log.debug("Skipping nested git repo", path=rel_str)return True@@ -479,7 +485,10 @@ class FileTraverser:"""packages: list[PackageInfo] = []seen_paths: set[str] = set()- prune_nested = not self._include_nested_repos+ # Mirrors GraphBuilder._prune_nested_git: when submodules or nested+ # repos are indexed, package-language/entry-point scans must not+ # prune them (both are `.git`-bearing subdirs to fs_walk).+ prune_nested = not (self._include_submodules or self._include_nested_repos)for depth in (1, 2):pattern = "/".join(["*"] * depth) + "/*"diff --git a/tests/unit/cli/test_update_include_submodules.py b/tests/unit/cli/test_update_include_submodules.pynew file mode 100644index 000000000..b5e900d6c--- /dev/null+++ b/tests/unit/cli/test_update_include_submodules.py@@ -0,0 +1,76 @@+"""`repowise update` must honor the persisted submodule flags.++A repo indexed with ``init --include-submodules`` records+``include_submodules: true`` in state.json; its incremental updates must+rebuild the graph with the same boundary semantics. ``_build_repo_graph``+previously constructed FileTraverser and GraphBuilder without the flags —+silently dropping submodule files (and their manifests) on every update.+Same class of bug as the ``git_tier`` gap (see test_update_git_tier.py).+"""++from __future__ import annotations++from repowise.cli.commands.update_cmd import _build_repo_graph, _rebuild_graph_and_git+++def _init_repo_with_initialized_submodule(tmp_path):+ """A parent repo containing an *initialized* submodule (`.git` file)."""+ import git as gitpython++ repo = gitpython.Repo.init(tmp_path)+ with repo.config_writer() as cw:+ cw.set_value("user", "name", "Alice")+ cw.set_value("user", "email", "[email protected]")+ (tmp_path / "a.py").write_text("x = 1\n")+ (tmp_path / ".gitmodules").write_text(+ '[submodule "libs/sub"]\n'+ " path = libs/sub\n"+ " url = https://github.com/example/sub.git\n"+ )+ sub = tmp_path / "libs" / "sub"+ sub.mkdir(parents=True)+ (sub / ".git").write_text("gitdir: ../../.git/modules/libs/sub\n")+ (sub / "mod.py").write_text("y = 2\n")+ repo.index.add(["a.py", ".gitmodules"])+ repo.index.commit("feat: add module a + submodule")+ repo.close()+++def _graph_paths(tmp_path, **kwargs) -> set[str]:+ parsed_files, _source_map, _builder, _structure, _count = _build_repo_graph(+ tmp_path, [], **kwargs+ )+ return {p.file_info.path for p in parsed_files}+++def test_update_graph_keeps_submodule_files_when_flag_set(tmp_path):+ """Equivalence with init: a submodule-indexed repo must keep its+ submodule files in the update-built graph."""+ _init_repo_with_initialized_submodule(tmp_path)++ paths = _graph_paths(tmp_path, include_submodules=True)++ assert "a.py" in paths+ assert "libs/sub/mod.py" in paths+++def test_update_graph_drops_submodule_files_by_default(tmp_path):+ """Legacy behavior: missing state key means submodules stay excluded."""+ _init_repo_with_initialized_submodule(tmp_path)++ paths = _graph_paths(tmp_path)++ assert "a.py" in paths+ assert "libs/sub/mod.py" not in paths+++def test_rebuild_threads_include_submodules(tmp_path):+ """The full incremental rebuild (graph + git re-index) honors the flag."""+ _init_repo_with_initialized_submodule(tmp_path)++ parsed_files, _sm, _gb, _rs, _fc, _gm = _rebuild_graph_and_git(+ tmp_path, [], {}, [], include_submodules=True+ )++ paths = {p.file_info.path for p in parsed_files}+ assert "libs/sub/mod.py" in pathsdiff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.pyindex 83276dfab..0e83d1be9 100644--- a/tests/unit/ingestion/test_traverser.py+++ b/tests/unit/ingestion/test_traverser.py@@ -473,6 +473,60 @@ class TestSubmoduleHandling:paths = [f.path for f in traverser.traverse()]assert any("libs/foo" in p for p in paths)+ def test_include_submodules_with_initialized_submodule(self, tmp_path: Path) -> None:+ """An *initialized* submodule carries a `.git` file — the nested-git+ boundary check must not override the explicit opt-in.++ Regression: ``include_submodules=True`` previously skipped parsing+ ``.gitmodules`` entirely, so initialized submodules fell through to+ the nested-git skip and were silently dropped anyway.+ """+ (tmp_path / ".gitmodules").write_text(+ '[submodule "libs/foo"]\n'+ " path = libs/foo\n"+ " url = https://github.com/example/foo.git\n"+ )+ (tmp_path / "libs" / "foo").mkdir(parents=True)+ (tmp_path / "libs" / "foo" / ".git").write_text("gitdir: ../../.git/modules/libs/foo\n")+ (tmp_path / "libs" / "foo" / "main.py").write_text("pass")+ traverser = FileTraverser(tmp_path, include_submodules=True)+ paths = [f.path for f in traverser.traverse()]+ assert any("libs/foo/main.py" in p for p in paths)+ assert traverser.stats.skipped_nested_repo == 0++ def test_initialized_submodule_skipped_by_default(self, tmp_path: Path) -> None:+ (tmp_path / ".gitmodules").write_text(+ '[submodule "libs/foo"]\n'+ " path = libs/foo\n"+ " url = https://github.com/example/foo.git\n"+ )+ (tmp_path / "libs" / "foo").mkdir(parents=True)+ (tmp_path / "libs" / "foo" / ".git").write_text("gitdir: ../../.git/modules/libs/foo\n")+ (tmp_path / "libs" / "foo" / "main.py").write_text("pass")+ traverser = FileTraverser(tmp_path)+ paths = [f.path for f in traverser.traverse()]+ assert not any("libs/foo" in p for p in paths)+ assert traverser.stats.skipped_submodule >= 1++ def test_include_submodules_keeps_other_nested_repos_skipped(self, tmp_path: Path) -> None:+ """The submodule opt-in must not widen to arbitrary nested repos."""+ (tmp_path / ".gitmodules").write_text(+ '[submodule "libs/foo"]\n'+ " path = libs/foo\n"+ " url = https://github.com/example/foo.git\n"+ )+ (tmp_path / "libs" / "foo").mkdir(parents=True)+ (tmp_path / "libs" / "foo" / ".git").write_text("gitdir: ../../.git/modules/libs/foo\n")+ (tmp_path / "libs" / "foo" / "main.py").write_text("pass")+ (tmp_path / "sibling_repo").mkdir()+ (tmp_path / "sibling_repo" / ".git").mkdir()+ (tmp_path / "sibling_repo" / "inner.py").write_text("pass")+ traverser = FileTraverser(tmp_path, include_submodules=True)+ paths = [f.path for f in traverser.traverse()]+ assert any("libs/foo/main.py" in p for p in paths)+ assert not any("sibling_repo" in p for p in paths)+ assert traverser.stats.skipped_nested_repo >= 1+def test_no_gitmodules_file(self, tmp_path: Path) -> None:(tmp_path / "app.py").write_text("pass")traverser = FileTraverser(tmp_path)