Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(coverage): drop files whose report lines no longer match the source at ingest (#4761)
Raghav Chamadiya3c8176c4793e
diff --git a/packages/cli/src/repowise/cli/commands/coverage_check_cmd.py b/packages/cli/src/repowise/cli/commands/coverage_check_cmd.pyindex 1ecd547d8..e4405bc34 100644--- a/packages/cli/src/repowise/cli/commands/coverage_check_cmd.py+++ b/packages/cli/src/repowise/cli/commands/coverage_check_cmd.py@@ -507,6 +507,9 @@ class _Reports:ignore=self.cfg.ignore,# The check stores nothing, so a fragment is judged, flagged partial.allow_partial=True,+ # A base report describes another commit than the files on disk,+ # so its lines cannot be checked against them.+ check_alignment=False,)diff --git a/packages/cli/src/repowise/cli/commands/coverage_cmd.py b/packages/cli/src/repowise/cli/commands/coverage_cmd.pyindex 4f8cdc2cd..094bd5c6d 100644--- a/packages/cli/src/repowise/cli/commands/coverage_cmd.py+++ b/packages/cli/src/repowise/cli/commands/coverage_cmd.py@@ -245,6 +245,8 @@ def coverage_add()for path, err in errors:console.print(f"[yellow] {path.name}: {err}[/yellow]")+ if stale := resolved.alignment.warning():+ console.print(f"[yellow]{escape(stale)}[/yellow]")# Counted from `resolved`, not from inside the `resolved.files`# branch below: total mapping loss leaves `files` empty, so# reading the count in there reported 0 report files unmapped@@ -368,6 +370,10 @@ def coverage_add()if not agg_matched and not map_records:+ if resolved.alignment.reasons:+ # Every mapped file was dropped as misaligned; the warning above says why.+ console.print("[red]Nothing was stored.[/red]")+ return Falseif len(errors) == len(report_paths) and not map_readable:# No report was readable as coverage; the lines above say why.console.print("[red]Nothing was stored.[/red]")diff --git a/packages/core/src/repowise/core/analysis/health/coverage/alignment.py b/packages/core/src/repowise/core/analysis/health/coverage/alignment.pynew file mode 100644index 000000000..adc19e461--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/coverage/alignment.py@@ -0,0 +1,116 @@+"""Does a report's line numbering still match the source on disk?++A report written before the source changed names lines that have since moved:+its executable lines land on blank lines, comments or past the end of the+file, and every coverage figure read from it describes code that is no longer+there. :func:`drop_misaligned` finds those files at ingest and keeps them out,+so they read as files without coverage.+"""++from __future__ import annotations++from collections.abc import Iterable+from dataclasses import dataclass, field+from pathlib import Path++from .model import FileCoverage++#: MISALIGNED_MAX_SHARE: a file whose coverable lines land on a blank line, a+#: comment or past EOF more often than this, and on at least+#: MISALIGNED_MIN_LINES of them, is dropped. A report taken from the current+#: source puts none there, so both only absorb edge cases in small files.+MISALIGNED_MAX_SHARE = 0.05+MISALIGNED_MIN_LINES = 3++# Comment prefixes for languages whose coverage tools report statement lines.+# Ceiling: Go coverprofiles and V8-based JS/TS reports mark every line of a+# block, blank and comment lines included, and Kotlin inline functions map to+# lines past EOF, so those files are not checked. Add a language here once its+# tools are known to report statement lines only. A bare ``*`` (block comment+# body) and PHP ``#`` are left out: code lines can start with them.+_COMMENT_PREFIXES: dict[str, tuple[str, ...]] = {+ ".py": ("#",),+ ".rb": ("#",),+ ".java": ("//", "/*"),+ ".cs": ("//", "/*"),+ ".php": ("//", "/*"),+}+++@dataclass+class Alignment:+ """Files dropped for misalignment, why, and the repo-wide line counts."""++ reasons: dict[str, str] = field(default_factory=dict)+ misaligned_lines: int = 0+ checked_lines: int = 0++ @property+ def share(self) -> float:+ return self.misaligned_lines / self.checked_lines if self.checked_lines else 0.0++ def warning(self) -> str | None:+ """One message naming the dropped files, ``None`` when none was dropped."""+ if not self.reasons:+ return None+ path, reason = next(iter(self.reasons.items()))+ cause = (+ "The report is likely older than the source: re-run the tests with "+ "coverage and add the new report."+ )+ if self.share > MISALIGNED_MAX_SHARE:+ cause = (+ f"{self.misaligned_lines} of {self.checked_lines} coverable lines "+ f"({self.share:.0%}) are off. {cause}"+ )+ return (+ f"Coverage for {len(self.reasons)} file(s) was not kept because the "+ f"report's lines no longer match the source (e.g. {path}: {reason}). {cause}"+ )+++def _misaligned(lines: Iterable[int], source: list[str], prefixes: tuple[str, ...]) -> int:+ count = 0+ for n in lines:+ text = source[n - 1].strip() if 0 < n <= len(source) else ""+ count += not text or text.startswith(prefixes)+ return count+++def _check(repo_root: Path, fc: FileCoverage) -> tuple[int, int] | None:+ """``(misaligned, coverable)`` line counts for *fc*, ``None`` when not checked."""+ prefixes = _COMMENT_PREFIXES.get(Path(fc.file_path).suffix.lower())+ lines = fc.coverable_lines or fc.covered_lines+ if not prefixes or not lines:+ return None+ try:+ source = (repo_root / fc.file_path).read_text(encoding="utf-8", errors="replace")+ except OSError:+ return None+ # Coverage tools number lines by newline only; ``splitlines`` also breaks+ # on form feeds and other separators a source file may contain.+ return _misaligned(lines, source.split("\n"), prefixes), len(lines)+++def drop_misaligned(+ repo_root: Path, files: list[FileCoverage]+) -> tuple[list[FileCoverage], Alignment]:+ """*files* without the ones whose line numbers no longer match the source."""+ kept: list[FileCoverage] = []+ result = Alignment()+ for fc in files:+ counts = _check(repo_root, fc)+ if counts is None:+ kept.append(fc)+ continue+ bad, total = counts+ result.misaligned_lines += bad+ result.checked_lines += total+ if bad >= MISALIGNED_MIN_LINES and bad / total > MISALIGNED_MAX_SHARE:+ result.reasons[fc.file_path] = (+ f"{bad} of {total} coverable lines land on blank or comment "+ "lines or past the end of the file"+ )+ else:+ kept.append(fc)+ return kept, resultdiff --git a/packages/core/src/repowise/core/analysis/health/coverage/discovery.py b/packages/core/src/repowise/core/analysis/health/coverage/discovery.pyindex 9661b2deb..bb1f6f6b1 100644--- a/packages/core/src/repowise/core/analysis/health/coverage/discovery.py+++ b/packages/core/src/repowise/core/analysis/health/coverage/discovery.py@@ -36,6 +36,7 @@ import pathspecfrom repowise.core.fs_walk import PRUNED_DIRS, WalkSnapshot, iter_glob+from .alignment import Alignment, drop_misalignedfrom .contexts import COVERAGE_DATA_FILE_NOTE, is_coverage_data_file, read_report_textfrom .detector import parse as parse_coveragefrom .model import (@@ -483,6 +484,8 @@ class ResolvedCoverage:# ingest against this rather than live HEAD so drift reads forward from the# right commit (#3901). ``None`` when nothing was read from disk.measured_at: datetime | None = None+ # Files dropped because the report's lines no longer match the source, and why.+ alignment: Alignment = field(default_factory=Alignment)@propertydef matched(self) -> int:@@ -1320,6 +1323,7 @@ def build_coverage_map(report_prefixes: Mapping[Path, str | None] | None = None,ignore: Sequence[str] = (),allow_partial: bool = False,+ check_alignment: bool = True,) -> tuple[ResolvedCoverage, list[tuple[Path, str]]]:"""Read + parse + resolve coverage reports end-to-end.@@ -1335,6 +1339,10 @@ def build_coverage_map(A partial mapping (:func:`mostly_unmapped`) keeps no coverage unless*allow_partial*; it comes back ``withheld`` so nothing stores or scores afragment as the repository's coverage.++ With *check_alignment*, a file whose report lines no longer match its+ source on disk is dropped (:func:`.alignment.drop_misaligned`) and listed in+ ``alignment``; pass ``False`` when the reports describe another commit."""prefixes = report_prefixes or {}parsed: list[CoverageReport] = []@@ -1375,6 +1383,10 @@ def build_coverage_map()if resolved.mapping_partial and not allow_partial:resolved.coverage_map, resolved.files, resolved.withheld = {}, [], True+ if check_alignment and resolved.files:+ resolved.files, resolved.alignment = drop_misaligned(Path(repo_root), resolved.files)+ for key in resolved.alignment.reasons:+ del resolved.coverage_map[key]# Only the reports that were actually read: one unreadable or empty# artifact must not drag the whole measurement back to its own date. A# withheld mapping contributed no rows at all, so it reports no time.diff --git a/packages/core/src/repowise/core/pipeline/incremental/loaders.py b/packages/core/src/repowise/core/pipeline/incremental/loaders.pyindex dfd5d41a1..6441c6e75 100644--- a/packages/core/src/repowise/core/pipeline/incremental/loaders.py+++ b/packages/core/src/repowise/core/pipeline/incremental/loaders.py@@ -323,5 +323,7 @@ async def _refresh_from_newer_reports(factory: Any, repo_path: Path, log: LogFn)provenance=resolved.provenance,)log(f"Re-read coverage from {len(reports)} report(s): {len(resolved.files)} file(s) stored")+ if stale := resolved.alignment.warning():+ log(f"[yellow]{stale}[/yellow]")except Exception as exc:log(f"[yellow]Coverage re-read skipped: {exc}[/yellow]")diff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.pyindex 7b461d06e..23f16d786 100644--- a/packages/core/src/repowise/core/pipeline/phases/analysis.py+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py@@ -241,6 +241,8 @@ def _build_pipeline_coverage()for path, err in errors:progress.on_message("warning", f" ↳ coverage {path.name}: {err}")+ if stale := resolved.alignment.warning():+ progress.on_message("warning", f" ↳ coverage: {stale}")return (resolved.coverage_map,diff --git a/tests/unit/health/test_coverage_alignment.py b/tests/unit/health/test_coverage_alignment.pynew file mode 100644index 000000000..9c7be7df8--- /dev/null+++ b/tests/unit/health/test_coverage_alignment.py@@ -0,0 +1,94 @@+"""Coverage ingest drops files whose report lines no longer match the source."""++from __future__ import annotations++from pathlib import Path++from repowise.core.analysis.health.coverage import build_coverage_map++_SOURCE = """import os+++def f(x):+ # double it+ y = x * 2+ return y+++def g():+ return os.sep+"""+# Statement lines of _SOURCE, as coverage.py would report them.+_ALIGNED = (1, 4, 6, 7, 10, 11)+++def _ingest(tmp_path: Path, lines: tuple[int, ...], **kwargs):+ (tmp_path / "src").mkdir()+ (tmp_path / "src" / "a.py").write_text(_SOURCE)+ (tmp_path / "src" / "b.ts").write_text("\n\n")+ da = "".join(f"DA:{n},1\n" for n in lines)+ report = tmp_path / "coverage.lcov"+ report.write_text(+ f"SF:src/a.py\n{da}end_of_record\nSF:src/b.ts\nDA:1,1\nDA:9,0\nend_of_record\n"+ )+ resolved, errors = build_coverage_map(+ tmp_path, [report], {"src/a.py", "src/b.ts"}, **kwargs+ )+ assert not errors+ return resolved+++def test_aligned_report_is_stored(tmp_path: Path) -> None:+ resolved = _ingest(tmp_path, _ALIGNED)+ assert "src/a.py" in resolved.coverage_map+ assert [fc.file_path for fc in resolved.files] == ["src/a.py", "src/b.ts"]+ assert resolved.alignment.reasons == {}+ assert resolved.alignment.warning() is None+++def test_shifted_report_is_dropped_with_a_reason(tmp_path: Path) -> None:+ # The same statements one line lower: mostly blank and comment lines now.+ resolved = _ingest(tmp_path, tuple(n + 1 for n in _ALIGNED))+ assert "src/a.py" not in resolved.coverage_map+ assert [fc.file_path for fc in resolved.files] == ["src/b.ts"]+ assert "blank or comment" in resolved.alignment.reasons["src/a.py"]+ assert "older than the source" in (resolved.alignment.warning() or "")+ # A language whose tools mark every line of a block is not checked.+ assert "src/b.ts" in resolved.coverage_map+++def test_lines_past_eof_are_dropped(tmp_path: Path) -> None:+ resolved = _ingest(tmp_path, (*_ALIGNED, 40, 41, 42))+ assert "src/a.py" not in resolved.coverage_map+ assert resolved.alignment.reasons["src/a.py"].startswith("3 of 9 ")+++def test_a_stray_line_or_two_is_tolerated(tmp_path: Path) -> None:+ resolved = _ingest(tmp_path, (*_ALIGNED, 40, 41))+ assert "src/a.py" in resolved.coverage_map+ assert resolved.alignment.warning() is None+++def test_form_feed_does_not_shift_lines(tmp_path: Path) -> None:+ # Coverage tools count newlines only; a form feed is not a line break.+ lines = ["a = 1", "x = 2 # \x0c", "b = 3", "", "c = 4", "", "d = 5", "", "e = 6"]+ (tmp_path / "c.py").write_text("\n".join(lines) + "\n")+ report = tmp_path / "c.lcov"+ report.write_text("SF:c.py\n" + "".join(f"DA:{n},1\n" for n in (1, 2, 3, 5, 7, 9)))+ resolved, _ = build_coverage_map(tmp_path, [report], {"c.py"})+ assert "c.py" in resolved.coverage_map+++def test_a_file_missing_from_disk_is_kept_unchecked(tmp_path: Path) -> None:+ # No source to compare with: the report is kept as it was before the check.+ report = tmp_path / "gone.lcov"+ report.write_text("SF:gone.py\n" + "".join(f"DA:{n},1\n" for n in (1, 2, 3)))+ resolved, _ = build_coverage_map(tmp_path, [report], {"gone.py"})+ assert "gone.py" in resolved.coverage_map+ assert resolved.alignment.reasons == {}+ assert resolved.alignment.checked_lines == 0+++def test_check_can_be_turned_off(tmp_path: Path) -> None:+ resolved = _ingest(tmp_path, (*_ALIGNED, 40, 41, 42), check_alignment=False)+ assert "src/a.py" in resolved.coverage_mapdiff --git a/tests/unit/pipeline/test_load_stored_coverage_map.py b/tests/unit/pipeline/test_load_stored_coverage_map.pyindex 5209cf705..f88cbf084 100644--- a/tests/unit/pipeline/test_load_stored_coverage_map.py+++ b/tests/unit/pipeline/test_load_stored_coverage_map.py@@ -118,6 +118,28 @@ async def test_loader_rereads_a_report_written_after_the_last_ingest(tmp_path, massert len(reads) == 1+async def test_a_reread_report_older_than_the_source_is_reported(tmp_path, monkeypatch):+ from types import SimpleNamespace++ repo_path = tmp_path / "repo"+ (repo_path / "src").mkdir(parents=True)+ # Lines 2-5 of the report now land on blank lines.+ (repo_path / "src" / "a.py").write_text("x = 1\n\n\n\n\n", encoding="utf-8")+ await _seed(repo_path, [{"file_path": "src/a.py", "line_coverage_pct": 0.0, "total_coverable_lines": 2}])+ report = repo_path / "coverage" / "lcov.info"+ report.parent.mkdir()+ report.write_text("SF:src/a.py\n" + "".join(f"DA:{n},1\n" for n in range(1, 6)) + "end_of_record\n")++ async def _metrics(_session, _repo_id):+ return [SimpleNamespace(file_path="src/a.py")]++ monkeypatch.setattr("repowise.core.persistence.crud.get_health_metrics", _metrics)+ logged: list[str] = []++ await load_stored_coverage_map(repo_path, log=logged.append)+ assert any("no longer match the source" in line for line in logged)++async def test_loader_keeps_stored_rows_when_reingest_is_off(tmp_path, monkeypatch):repo_path = tmp_path / "repo"repo_path.mkdir()