Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(health): make `module` the package boundary, not a graph community label (#1359)

Raghav Chamadiyaa3b6714c5523
diff --git a/packages/core/src/repowise/core/analysis/health/engine.py b/packages/core/src/repowise/core/analysis/health/engine.py
index 1166cb303..7ec4baa23 100644
--- a/packages/core/src/repowise/core/analysis/health/engine.py
+++ b/packages/core/src/repowise/core/analysis/health/engine.py
@@ -86,7 +86,7 @@ def _log_duplication_diagnostics(report: DuplicationReport) -> None:
def _fallback_module(rel_path: str) -> str | None:
- """Top-level directory as a stand-in module label when no community map.
+ """Top-level directory as a stand-in module label.
Returns ``None`` for root-level files so the rollup endpoint doesn't
create a phantom "" bucket.
@@ -98,6 +98,81 @@ def _fallback_module(rel_path: str) -> str | None:
return head or None
+# Files that mark a directory as the root of a distributable package.
+#
+# Every name here is one the traverser actually emits. That is the binding
+# constraint, not the set of manifests that exist: the analyzer sees
+# ``parsed_files``, and the traverser drops any file whose language it cannot
+# detect. Verified by running ``FileTraverser`` over a directory holding one of
+# each — ``go.mod``, ``pom.xml``, ``build.gradle``, ``Gemfile``, ``setup.cfg``
+# and ``*.csproj`` are all dropped, so listing them here would be dead entries
+# that read as support. Go, Maven, Groovy-Gradle, Ruby and .NET repos therefore
+# fall back to the top-level directory, which is exactly what they get today —
+# no regression, just no improvement until the traverser yields those files.
+#
+# Kept conservative besides: a false root fragments the rollup, and the
+# top-level fallback is already reasonable. ``requirements.txt`` is deliberately
+# absent — it appears in subdirectories that are not packages.
+_PACKAGE_MANIFESTS = frozenset(
+ {
+ "package.json",
+ "pyproject.toml",
+ "setup.py",
+ "Cargo.toml",
+ "build.gradle.kts",
+ "composer.json",
+ }
+)
+
+
+def _package_roots(all_paths: set[str]) -> set[str]:
+ """Directories holding a package manifest, from the analyzed file list.
+
+ A manifest at the repo root is ignored: it would put every file in one
+ bucket, which is the degenerate case this exists to avoid.
+ """
+ roots: set[str] = set()
+ for p in all_paths:
+ norm = p.replace("\\", "/")
+ if "/" not in norm:
+ continue
+ head, base = norm.rsplit("/", 1)
+ if base in _PACKAGE_MANIFESTS:
+ roots.add(head)
+ return roots
+
+
+def _module_for(rel_path: str, package_roots: set[str]) -> str | None:
+ """The package boundary a file belongs to.
+
+ ``module`` is the directory/package axis and nothing else. It used to be
+ ``community_label or top_level_dir``, which mixed two namespaces in one
+ field: graph community labels are semantic clusters named after a member
+ directory, so a file could report a module it does not live in (measured on
+ this repo: 1,355 of 3,263 files, e.g. every ``packages/api-client`` source
+ file reporting ``tests/unit``). Worse, only the full-index path ever passed
+ a community map — the incremental, re-score and ``repowise health`` paths
+ did not — so which namespace a row carried depended on which code path last
+ wrote it.
+
+ The top-level directory alone is not enough either: on a monorepo it buckets
+ 69% of this repo under ``packages``. So prefer the deepest enclosing package
+ manifest and fall back to the top-level directory, which is exactly the old
+ behaviour on a repo with no nested packages.
+ """
+ norm = rel_path.replace("\\", "/")
+ if "/" not in norm:
+ return None
+ if package_roots:
+ parts = norm.split("/")
+ # Deepest first, so a nested package wins over the one containing it.
+ for i in range(len(parts) - 1, 0, -1):
+ candidate = "/".join(parts[:i])
+ if candidate in package_roots:
+ return candidate
+ return _fallback_module(norm)
+
+
def _read_source_lines(abs_path: str) -> list[str] | None:
"""Read a file's source as 1-indexed lines for the Extract Helper snippet.
@@ -330,7 +405,12 @@ class HealthAnalyzer:
# PageRank is optional — graph_builder.symbol_pagerank exists but
# is symbol-level; we use file-level in-degree as the dependents
# signal (cheap, deterministic, conservative).
- path_basenames = _path_basenames({pf.file_info.path for pf in self.parsed_files})
+ analyzed_paths = {pf.file_info.path for pf in self.parsed_files}
+ path_basenames = _path_basenames(analyzed_paths)
+ # One scan of the file list decides the package boundaries for every
+ # file, so ``module`` is a property of the repo layout rather than of
+ # whichever code path happens to be running.
+ package_roots = _package_roots(analyzed_paths)
repo_commit_counts = _build_repo_commit_counts(self.git_meta_map)
graph_view: HasEdge | None = _ImportEdgeView(self.graph) if self.graph is not None else None
@@ -417,6 +497,7 @@ class HealthAnalyzer:
pf,
fcx,
path_basenames,
+ package_roots,
disabled=file_disabled,
dup_report=dup_report,
graph_view=graph_view,
@@ -490,7 +571,12 @@ class HealthAnalyzer:
)
changed_set: set[str] | None = set(changed_files) if changed_files is not None else None
- path_basenames = _path_basenames({pf.file_info.path for pf in self.parsed_files})
+ analyzed_paths = {pf.file_info.path for pf in self.parsed_files}
+ path_basenames = _path_basenames(analyzed_paths)
+ # One scan of the file list decides the package boundaries for every
+ # file, so ``module`` is a property of the repo layout rather than of
+ # whichever code path happens to be running.
+ package_roots = _package_roots(analyzed_paths)
repo_commit_counts = _build_repo_commit_counts(self.git_meta_map)
graph_view: HasEdge | None = _ImportEdgeView(self.graph) if self.graph is not None else None
@@ -591,6 +677,7 @@ class HealthAnalyzer:
pf,
fcx,
path_basenames,
+ package_roots,
disabled=file_disabled,
dup_report=dup_report,
graph_view=graph_view,
@@ -767,6 +854,7 @@ class HealthAnalyzer:
pf: Any,
fcx: FileComplexity,
path_basenames: set[str],
+ package_roots: set[str],
*,
disabled: list[str],
dup_report: DuplicationReport,
@@ -815,7 +903,7 @@ class HealthAnalyzer:
clones = dup_report.pairs_by_file.get(file_path, [])
dup_pct = dup_report.duplication_pct.get(file_path)
- module = self.module_map.get(file_path) or _fallback_module(file_path)
+ module = _module_for(file_path, package_roots)
file_git_meta = self.git_meta_map.get(file_path, {}) or {}
blame_idx_obj = file_git_meta.get("blame_index")
diff --git a/tests/unit/health/test_file_nloc.py b/tests/unit/health/test_file_nloc.py
index 645a39832..469c76f91 100644
--- a/tests/unit/health/test_file_nloc.py
+++ b/tests/unit/health/test_file_nloc.py
@@ -91,6 +91,7 @@ def test_health_metric_nloc_uses_file_nloc():
pf,
fcx,
path_basenames={"route.js"},
+ package_roots=set(),
disabled=[],
dup_report=DuplicationReport(),
)
diff --git a/tests/unit/health/test_module_attribution.py b/tests/unit/health/test_module_attribution.py
new file mode 100644
index 000000000..1d8d78f70
--- /dev/null
+++ b/tests/unit/health/test_module_attribution.py
@@ -0,0 +1,159 @@
+"""``module`` is the package boundary, and every write path agrees on it.
+
+``module`` used to be ``community_label or top_level_dir``. Two namespaces in
+one field, and only the full-index path ever supplied the community map — the
+incremental, re-score and ``repowise health`` paths did not — so a file's module
+depended on which code path last wrote its row. Measured on this repo before the
+change: 1,355 of 3,263 files reported a module they do not live in.
+"""
+
+from __future__ import annotations
+
+from repowise.core.analysis.health.engine import (
+ _PACKAGE_MANIFESTS,
+ _fallback_module,
+ _module_for,
+ _package_roots,
+)
+
+
+def test_package_roots_are_read_from_manifests_in_the_file_list():
+ roots = _package_roots(
+ {
+ "packages/core/pyproject.toml",
+ "packages/core/src/thing.py",
+ "packages/ui/package.json",
+ "packages/ui/src/thing.ts",
+ "docs/guide.md",
+ }
+ )
+ assert roots == {"packages/core", "packages/ui"}
+
+
+def test_every_listed_manifest_is_one_the_traverser_actually_emits():
+ """The list is bounded by what reaches ``parsed_files``, not by what exists.
+
+ The analyzer only ever sees traversed files, and the traverser drops any
+ file whose language it cannot detect — ``go.mod``, ``pom.xml``,
+ ``build.gradle``, ``Gemfile`` and ``setup.cfg`` among them. Listing those
+ would be dead entries that read as support for Go, Maven and Ruby
+ monorepos. This pins the constraint so the list cannot quietly grow past it.
+ """
+ import tempfile
+ from pathlib import Path
+
+ from repowise.core.ingestion.traverser import FileTraverser
+
+ root = Path(tempfile.mkdtemp())
+ for i, name in enumerate(sorted(_PACKAGE_MANIFESTS)):
+ pkg = root / f"pkg{i}"
+ pkg.mkdir()
+ (pkg / name).write_text("{}\n" if name.endswith("json") else "x = 1\n", encoding="utf-8")
+
+ emitted = {p.path.replace("\\", "/").rsplit("/", 1)[-1] for p in FileTraverser(root).traverse()}
+ assert emitted >= _PACKAGE_MANIFESTS, sorted(_PACKAGE_MANIFESTS - emitted)
+
+
+def test_a_repo_root_manifest_is_not_a_package_root():
+ """Otherwise every file in a single-package repo lands in one bucket.
+
+ That is the degenerate case the package-root rule exists to avoid, and it
+ is the common shape: a plain app repo with one ``package.json`` at the top.
+ """
+ assert _package_roots({"package.json", "src/a.ts", "src/b.ts"}) == set()
+
+
+def test_module_is_the_package_not_the_top_level_directory():
+ """The monorepo case. ``packages`` is a container, not a module.
+
+ Taking the first path segment put 69% of this repo in one bucket, which is
+ what made per-module averages and ``module:`` targeting useless here.
+ """
+ roots = {"packages/core", "packages/ui"}
+ assert _module_for("packages/core/src/deep/nested/thing.py", roots) == "packages/core"
+ assert _module_for("packages/ui/src/thing.ts", roots) == "packages/ui"
+
+
+def test_the_deepest_enclosing_package_wins():
+ roots = {"packages/core", "packages/core/vendor/inner"}
+ assert _module_for("packages/core/vendor/inner/x.py", roots) == "packages/core/vendor/inner"
+ assert _module_for("packages/core/src/x.py", roots) == "packages/core"
+
+
+def test_a_file_outside_every_package_falls_back_to_the_top_level_directory():
+ roots = {"packages/core"}
+ assert _module_for("tests/unit/test_x.py", roots) == "tests"
+ assert _module_for("docs/guide.md", roots) == "docs"
+
+
+def test_no_manifests_anywhere_is_exactly_the_old_behaviour():
+ """Repos with no nested packages must not move.
+
+ Measured on the two sibling repos in this workspace (a FastAPI backend and
+ a Next.js frontend): zero nested manifests, so both keep the attribution
+ they already had.
+ """
+ for path in ("app/routers/files.py", "src/components/x.tsx", "tests/unit/a.py"):
+ assert _module_for(path, set()) == _fallback_module(path)
+
+
+def test_root_level_files_have_no_module():
+ """``None``, not ``""`` — the rollup must not grow a phantom empty bucket."""
+ assert _module_for("README.md", {"packages/core"}) is None
+ assert _module_for("Makefile", set()) is None
+
+
+def test_windows_separators_resolve_to_the_same_module():
+ roots = {"packages/core"}
+ assert _module_for("packages\\core\\src\\thing.py", roots) == "packages/core"
+
+
+def _analyze(tmp_path, *, module_map):
+ """Run the real analyzer over a two-package tree and return {path: module}."""
+ from repowise.core.analysis.health import HealthAnalyzer
+ from repowise.core.ingestion.parser import parse_file
+ from repowise.core.ingestion.traverser import FileTraverser
+
+ (tmp_path / "packages" / "core").mkdir(parents=True, exist_ok=True)
+ (tmp_path / "packages" / "ui").mkdir(parents=True, exist_ok=True)
+ (tmp_path / "packages" / "core" / "pyproject.toml").write_text(
+ '[project]\nname = "core"\n', encoding="utf-8"
+ )
+ (tmp_path / "packages" / "core" / "thing.py").write_text("a = 1\n", encoding="utf-8")
+ (tmp_path / "packages" / "ui" / "pyproject.toml").write_text(
+ '[project]\nname = "ui"\n', encoding="utf-8"
+ )
+ (tmp_path / "packages" / "ui" / "widget.py").write_text("b = 2\n", encoding="utf-8")
+
+ parsed = [
+ parse_file(f, (tmp_path / f.path).read_bytes())
+ for f in FileTraverser(tmp_path).traverse()
+ ]
+ report = HealthAnalyzer(None, parsed_files=parsed, module_map=module_map).analyze()
+ return {m.file_path: m.module for m in report.metrics}
+
+
+def test_the_engine_wires_real_package_roots_through_to_the_metric(tmp_path):
+ """End to end, not just the helper: the stored ``module`` is the package.
+
+ The helpers can be right while the call site still reads the old value, so
+ this asserts on what the analyzer actually writes.
+ """
+ modules = _analyze(tmp_path, module_map=None)
+ assert modules["packages/core/thing.py"] == "packages/core"
+ assert modules["packages/ui/widget.py"] == "packages/ui"
+
+
+def test_a_community_map_can_no_longer_change_the_answer(tmp_path):
+ """The regression that motivated this, as an executable invariant.
+
+ Only the full-index path ever passed a ``module_map``; the incremental,
+ re-score and ``repowise health`` paths did not, so a row's namespace
+ depended on which one last wrote it. Supplying a hostile map must now be
+ inert — this is what makes the four paths agree.
+ """
+ poisoned = {
+ "packages/core/thing.py": "tests/unit",
+ "packages/ui/widget.py": "repowise/distill (7)",
+ }
+ assert _analyze(tmp_path, module_map=poisoned) == _analyze(tmp_path, module_map=None)
diff --git a/tests/unit/health/test_severity_overrides_integration.py b/tests/unit/health/test_severity_overrides_integration.py
index 088086933..0d7bd4df9 100644
--- a/tests/unit/health/test_severity_overrides_integration.py
+++ b/tests/unit/health/test_severity_overrides_integration.py
@@ -61,6 +61,7 @@ def _evaluate(severity_overrides):
pf,
fcx,
path_basenames=set(),
+ package_roots=set(),
disabled=[],
dup_report=DuplicationReport(),
severity_overrides=severity_overrides,