Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(coverage): coverage-backed missing-test signal (test gaps proven by the per-test map) (#793)

Raghav Chamadiya9b4d39237ba0
diff --git a/packages/core/src/repowise/core/analysis/missing_test_signal.py b/packages/core/src/repowise/core/analysis/missing_test_signal.py
new file mode 100644
index 000000000..7f34a8e60
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/missing_test_signal.py
@@ -0,0 +1,146 @@
+"""Coverage-backed missing-test signal.
+
+A filename path-pattern "you changed X but didn't test it" check knows only that
+a test *exists* for a file, never whether the committed change is *covered* by
+it. Only a real per-test coverage map can tell the two apart. This module mines
+that map (built by ``repowise coverage add``) into two PR-time sub-signals,
+computed per changed file from the change's changed *line numbers*:
+
+ 1. UNTESTED CHANGE (the strong signal): the file IS in the coverage map, but
+ the changed lines intersect no test's covered set. Some test exercises the
+ file, yet these specific lines are uncovered. Fires ONLY when the file has
+ coverage rows: a file with zero rows is "no coverage data / unknown", never
+ "untested". Conflating the two is exactly the dishonesty that makes
+ filename-guessing worthless, so the two are kept strictly apart here.
+
+ 2. STALE TEST CANDIDATE (the weak signal): the changed lines ARE covered, but
+ none of the covering test *files* appear in the diff - covered code changed
+ without its guarding test being touched. Degrades honestly when a covering
+ row carries no test file path (coverage.py dynamic-context ids like
+ ``mod.test_add`` have no path): such rows can neither confirm nor deny a
+ stale test, so they never fire the signal on their own.
+
+The detector is a pure classifier over ``(session, repo_id, changed-lines
+map)`` - the inverse view of ``repowise impacted-tests`` over the same map. It
+is split from any surface (the ``impacted_tests`` command resolver does the
+same) so the diff -> lines -> signal path is testable against a seeded
+``test_coverage`` table.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class UntestedChange:
+ """A file whose changed lines no recorded test covers (strong signal)."""
+
+ source_file: str
+ uncovered_lines: list[int]
+ changed_line_count: int
+
+
+@dataclass
+class StaleTestCandidate:
+ """Covered changed lines whose covering test file(s) are not in the diff."""
+
+ source_file: str
+ covering_test_files: list[str]
+ # Covering rows that carry no test-file path (dynamic-context ids). Reported
+ # for transparency; they do not by themselves fire the signal.
+ covering_test_ids_without_file: list[str] = field(default_factory=list)
+
+
+@dataclass
+class MissingTestReport:
+ """Buckets from classifying each changed file against the coverage map."""
+
+ #: file has coverage rows but the changed lines are covered by no test.
+ untested_changes: list[UntestedChange] = field(default_factory=list)
+ #: changed lines covered, but no covering test file is in the diff.
+ stale_test_candidates: list[StaleTestCandidate] = field(default_factory=list)
+ #: changed lines covered and a covering test file IS in the diff (or the
+ #: covering rows are pathless so staleness is unknowable) - no signal.
+ covered: list[str] = field(default_factory=list)
+ #: file has ZERO coverage rows - unknown, NOT untested. Never a warning.
+ no_data: list[str] = field(default_factory=list)
+ #: true when the repo has no per-test map at all (nothing to say honestly).
+ map_empty: bool = False
+
+ def has_signal(self) -> bool:
+ return bool(self.untested_changes or self.stale_test_candidates)
+
+
+async def detect_missing_tests(
+ session,
+ repository_id: str,
+ changed: dict[str, set[int]],
+) -> MissingTestReport:
+ """Classify each changed file into the missing-test buckets.
+
+ *changed* is ``{source_file: {changed line numbers}}`` (the new side of a
+ diff, from :func:`repowise.core.analysis.changed_lines.changed_lines`). Its
+ keys are also the set of files touched by the change - used to decide
+ whether a covering test file was itself edited (sub-signal 2).
+ """
+ from repowise.core.persistence.crud import get_test_coverage_summary, tests_covering
+
+ report = MissingTestReport()
+ if not changed:
+ return report
+
+ summary = await get_test_coverage_summary(session, repository_id)
+ if summary.get("pair_count", 0) == 0:
+ # No map ingested: we cannot honestly say anything about coverage.
+ report.map_empty = True
+ return report
+
+ changed_files = set(changed.keys())
+
+ for source_file, lines in sorted(changed.items()):
+ # Does the file appear in the map at all? (lines=None -> every test that
+ # touches the file, regardless of which lines). Empty => no data.
+ all_rows = await tests_covering(session, repository_id, source_file, lines=None)
+ if not all_rows:
+ report.no_data.append(source_file)
+ continue
+
+ # The file IS in the map. Do any tests cover the CHANGED lines?
+ hit_rows = await tests_covering(session, repository_id, source_file, lines=lines)
+ if not hit_rows:
+ # Covered file, uncovered change: the strong signal.
+ covered_here: set[int] = set()
+ for r in all_rows:
+ covered_here.update(r["covered_lines"])
+ uncovered = sorted(lines - covered_here)
+ report.untested_changes.append(
+ UntestedChange(
+ source_file=source_file,
+ uncovered_lines=uncovered,
+ changed_line_count=len(lines),
+ )
+ )
+ continue
+
+ # Changed lines ARE covered. Was a covering test touched in the diff?
+ covering_test_files = sorted({r["test_file"] for r in hit_rows if r["test_file"]})
+ pathless_ids = sorted({r["test_id"] for r in hit_rows if not r["test_file"]})
+ touched = [tf for tf in covering_test_files if tf in changed_files]
+ if touched:
+ report.covered.append(source_file)
+ elif covering_test_files:
+ # Every covering test file we can name is absent from the diff.
+ report.stale_test_candidates.append(
+ StaleTestCandidate(
+ source_file=source_file,
+ covering_test_files=covering_test_files,
+ covering_test_ids_without_file=pathless_ids,
+ )
+ )
+ else:
+ # Only pathless covering rows: covered, but staleness is unknowable.
+ # Degrade honestly - do not guess a stale test.
+ report.covered.append(source_file)
+
+ return report
diff --git a/packages/core/src/repowise/core/analysis/pr_blast.py b/packages/core/src/repowise/core/analysis/pr_blast.py
index c1f45738a..01b4989be 100644
--- a/packages/core/src/repowise/core/analysis/pr_blast.py
+++ b/packages/core/src/repowise/core/analysis/pr_blast.py
@@ -229,15 +229,24 @@ class PRBlastRadiusAnalyzer:
return reviewers[:5]
async def _find_test_gaps(self, affected_files: list[str]) -> list[str]:
- """Return files that lack a corresponding test file.
-
- Checks graph_nodes for paths matching test_<name>, <name>_test, or
- <name>.spec.* patterns. Test files themselves are excluded — they
+ """Return files that lack a test, coverage-backed where the map has data.
+
+ A file with >=1 per-test coverage row (from ``repowise coverage add``) is
+ coverage-*proven* to be exercised by some test, so it is never a gap -
+ this supersedes the filename guess for files the map can speak to. Where
+ the per-test map has no data for a file, fall back to the filename
+ pattern (test_<name>, <name>_test, <name>.spec.*) - an honest "unknown",
+ never asserted as untested. Test files themselves are excluded; they
don't need their own tests.
"""
if not affected_files:
return []
+ from repowise.core.persistence.crud import covered_source_files
+
+ # Coverage-proven-tested files: absent from gaps regardless of naming.
+ covered = await covered_source_files(self._session, self._repo_id, set(affected_files))
+
node_res = await self._session.execute(
select(GraphNode.node_id, GraphNode.is_test).where(
GraphNode.repository_id == self._repo_id,
@@ -247,7 +256,7 @@ class PRBlastRadiusAnalyzer:
# Build a set of affected files that are themselves test files
test_file_set = {row[0] for row in node_res.all() if row[1]}
- # Fetch all test paths for matching
+ # Fetch all test paths for the filename fallback (map-no-data files only)
all_test_res = await self._session.execute(
select(GraphNode.node_id).where(
GraphNode.repository_id == self._repo_id,
@@ -261,6 +270,9 @@ class PRBlastRadiusAnalyzer:
# Skip test files — they don't need their own tests
if path in test_file_set:
continue
+ # Coverage proves a test exercises this file: not a gap.
+ if path in covered:
+ continue
base = os.path.splitext(os.path.basename(path))[0]
ext = os.path.splitext(path)[1].lstrip(".")
has_test = any(
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py
index 32ed8e674..b97ed713a 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py
@@ -15,6 +15,7 @@ from .coverage import (
save_coverage_files,
)
from .coverage_map import (
+ covered_source_files,
files_covered_by,
get_test_coverage_summary,
save_test_coverage,
@@ -52,6 +53,7 @@ from .refactoring import (
__all__ = [
"HEALTH_SNAPSHOT_RETENTION",
+ "covered_source_files",
"files_covered_by",
"get_coverage_summary",
"get_dead_code_findings",
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/coverage_map.py b/packages/core/src/repowise/core/persistence/crud/analysis/coverage_map.py
index 79f2450b3..67238ba5f 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/coverage_map.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/coverage_map.py
@@ -135,6 +135,30 @@ async def tests_covering(
return out
+async def covered_source_files(
+ session: AsyncSession,
+ repository_id: str,
+ source_files: set[str],
+) -> set[str]:
+ """Return the subset of *source_files* that have >=1 per-test coverage row.
+
+ A file with a row is coverage-*proven* to be exercised by some test - the
+ honest basis for "this changed file is tested". A file with no row is
+ "unknown" (the coverage run may simply not have exercised it), never a
+ proof of absence, so callers fall back to a labelled filename guess for
+ those rather than asserting a test gap.
+ """
+ if not source_files:
+ return set()
+ result = await session.execute(
+ select(TestCoverageEntry.source_file).where(
+ TestCoverageEntry.repository_id == repository_id,
+ TestCoverageEntry.source_file.in_(source_files),
+ )
+ )
+ return {row[0] for row in result.all()}
+
+
async def files_covered_by(
session: AsyncSession,
repository_id: str,
diff --git a/packages/server/src/repowise/server/mcp_server/tool_risk/assessment.py b/packages/server/src/repowise/server/mcp_server/tool_risk/assessment.py
index 7aadad339..f995febf3 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_risk/assessment.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_risk/assessment.py
@@ -163,12 +163,18 @@ def _compute_impact_surface(
async def _check_test_gap(session: AsyncSession, repo_id: str, target: str) -> bool:
- """Return True if no test file corresponding to *target* exists in graph_nodes.
+ """Return True if *target* has no test, coverage-backed where the map has data.
- Test files themselves (is_test=True) are never considered to have a test gap.
+ A file with a per-test coverage row (from ``repowise coverage add``) is
+ coverage-*proven* tested, so it is never a gap. Where the map has no data for
+ the file, fall back to the filename pattern (test_<name>, <name>_test,
+ <name>.spec.*) - an honest "unknown", never asserted as untested. Test files
+ themselves (is_test=True) are never a gap.
"""
import os
+ from repowise.core.persistence.crud import covered_source_files
+
# Test files don't need tests — skip the check entirely
node_res = await session.execute(
select(GraphNode.is_test)
@@ -182,6 +188,10 @@ async def _check_test_gap(session: AsyncSession, repo_id: str, target: str) -> b
if row is True:
return False
+ # Coverage proves a test exercises this file: not a gap.
+ if await covered_source_files(session, repo_id, {target}):
+ return False
+
base = os.path.splitext(os.path.basename(target))[0]
ext = os.path.splitext(target)[1].lstrip(".")
# Build a LIKE pattern broad enough to catch test_<base>, <base>_test, <base>.spec.*
diff --git a/tests/unit/persistence/test_missing_test_signal.py b/tests/unit/persistence/test_missing_test_signal.py
new file mode 100644
index 000000000..6732eb2c1
--- /dev/null
+++ b/tests/unit/persistence/test_missing_test_signal.py
@@ -0,0 +1,140 @@
+"""The coverage-backed missing-test detector: changed lines -> two sub-signals.
+
+Exercises :func:`detect_missing_tests` against a seeded ``test_coverage`` table
+so the four honest outcomes are pinned:
+ * UNTESTED CHANGE - file in the map, changed lines covered by no test;
+ * STALE TEST CANDIDATE - covered lines, covering test file not in the diff;
+ * NO DATA - file has zero coverage rows (unknown, never "untested");
+ * COVERED - covered lines and the covering test file IS in the diff.
+
+Plus the honesty guards: an empty map says nothing, and pathless covering rows
+(dynamic-context ids with no test file) never fire the stale signal.
+"""
+
+from __future__ import annotations
+
+from repowise.core.analysis.health.coverage import TestCoverage
+from repowise.core.analysis.missing_test_signal import detect_missing_tests
+from repowise.core.persistence.crud import save_test_coverage
+from tests.unit.persistence.helpers import insert_repo
+
+
+def _rec(test_id: str, source_file: str, lines: list[int], test_file: str | None):
+ return TestCoverage(
+ test_id=test_id,
+ file_path=source_file,
+ covered_lines=lines,
+ source_format="coverage.py",
+ test_file=test_file,
+ )
+
+
+async def _seed(async_session, records):
+ repo = await insert_repo(async_session)
+ await save_test_coverage(async_session, repo.id, records, source_format="coverage.py")
+ await async_session.commit()
+ return repo
+
+
+async def test_untested_change_fires_when_file_covered_but_lines_are_not(async_session):
+ # foo.py is exercised (lines 1-3, 8-9), but the change touches line 99.
+ repo = await _seed(
+ async_session,
+ [
+ _rec("tests/test_foo.py::test_a", "src/foo.py", [1, 2, 3], "tests/test_foo.py"),
+ _rec("tests/test_foo.py::test_b", "src/foo.py", [8, 9], "tests/test_foo.py"),
+ ],
+ )
+ report = await detect_missing_tests(async_session, repo.id, {"src/foo.py": {99}})
+
+ assert len(report.untested_changes) == 1
+ uc = report.untested_changes[0]
+ assert uc.source_file == "src/foo.py"
+ assert uc.uncovered_lines == [99]
+ assert uc.changed_line_count == 1
+ assert report.stale_test_candidates == []
+ assert report.no_data == []
+ assert report.covered == []
+
+
+async def test_stale_test_candidate_fires_when_covering_test_not_in_diff(async_session):
+ repo = await _seed(
+ async_session,
+ [_rec("tests/test_foo.py::test_a", "src/foo.py", [1, 2, 3], "tests/test_foo.py")],
+ )
+ # Change touches covered line 2, but the diff does NOT include the test file.
+ report = await detect_missing_tests(async_session, repo.id, {"src/foo.py": {2}})
+
+ assert len(report.stale_test_candidates) == 1
+ stc = report.stale_test_candidates[0]
+ assert stc.source_file == "src/foo.py"
+ assert stc.covering_test_files == ["tests/test_foo.py"]
+ assert report.untested_changes == []
+ assert report.covered == []
+
+
+async def test_covered_when_test_file_is_in_the_diff(async_session):
+ repo = await _seed(
+ async_session,
+ [_rec("tests/test_foo.py::test_a", "src/foo.py", [1, 2, 3], "tests/test_foo.py")],
+ )
+ # Both the source AND its covering test file are in the diff -> no signal.
+ report = await detect_missing_tests(
+ async_session, repo.id, {"src/foo.py": {2}, "tests/test_foo.py": {10}}
+ )
+
+ assert report.covered == ["src/foo.py"]
+ assert report.untested_changes == []
+ assert report.stale_test_candidates == []
+
+
+async def test_no_data_file_is_never_untested(async_session):
+ # The map has data for foo.py, but the change touches bar.py (zero rows).
+ repo = await _seed(
+ async_session,
+ [_rec("tests/test_foo.py::test_a", "src/foo.py", [1, 2, 3], "tests/test_foo.py")],
+ )
+ report = await detect_missing_tests(async_session, repo.id, {"src/bar.py": {5}})
+
+ assert report.no_data == ["src/bar.py"]
+ assert report.untested_changes == []
+ assert report.stale_test_candidates == []
+
+
+async def test_empty_map_says_nothing(async_session):
+ repo = await insert_repo(async_session)
+ report = await detect_missing_tests(async_session, repo.id, {"src/foo.py": {1}})
+
+ assert report.map_empty is True
+ assert not report.has_signal()
+ assert report.no_data == []
+
+
+async def test_pathless_covering_rows_do_not_fire_stale(async_session):
+ # coverage.py dynamic-context ids carry no test file path: covered, but we
+ # cannot tell whether the guarding test was touched -> never "stale".
+ repo = await _seed(
+ async_session,
+ [_rec("foo.test_add", "src/foo.py", [1, 2, 3], None)],
+ )
+ report = await detect_missing_tests(async_session, repo.id, {"src/foo.py": {2}})
+
+ assert report.covered == ["src/foo.py"]
+ assert report.stale_test_candidates == []
+ assert report.untested_changes == []
+
+
+async def test_uncovered_lines_are_narrowed_to_the_change(async_session):
+ # Change touches lines 2 (covered) and 99 (not) - since at least one line is
+ # covered the file is NOT untested; it is a stale-test candidate, and the
+ # covered set is what protects it.
+ repo = await _seed(
+ async_session,
+ [_rec("tests/test_foo.py::test_a", "src/foo.py", [1, 2, 3], "tests/test_foo.py")],
+ )
+ report = await detect_missing_tests(async_session, repo.id, {"src/foo.py": {2, 99}})
+
+ # A partial hit still counts as covered (some test touches the change), so
+ # this is a stale-test candidate, not an untested change.
+ assert report.untested_changes == []
+ assert len(report.stale_test_candidates) == 1
diff --git a/tests/unit/persistence/test_pr_blast_coverage_gaps.py b/tests/unit/persistence/test_pr_blast_coverage_gaps.py
new file mode 100644
index 000000000..eaafd3640
--- /dev/null
+++ b/tests/unit/persistence/test_pr_blast_coverage_gaps.py
@@ -0,0 +1,67 @@
+"""Coverage-backed test-gap detection in PRBlastRadiusAnalyzer._find_test_gaps.
+
+Coverage-backed gaps: a file with a per-test coverage row is coverage-proven tested
+(never a gap, even if no test file matches its NAME); a file the map has no data
+for falls back to the filename pattern (honest "unknown", never asserted
+untested). Seeds graph_nodes + a test_coverage row and pins both paths.
+"""
+
+from __future__ import annotations
+
+from repowise.core.analysis.health.coverage import TestCoverage
+from repowise.core.analysis.pr_blast import PRBlastRadiusAnalyzer
+from repowise.core.persistence.crud import save_test_coverage
+from repowise.core.persistence.models import GraphNode
+from tests.unit.persistence.helpers import insert_repo
+
+
+async def _node(session, repo_id, node_id, *, is_test=False):
+ session.add(GraphNode(repository_id=repo_id, node_id=node_id, is_test=is_test))
+
+
+async def test_coverage_proven_file_is_not_a_gap(async_session):
+ repo = await insert_repo(async_session)
+ # weirdly.py is covered by a test whose NAME does not match it - the
+ # filename heuristic would wrongly call it a gap; coverage clears it.
+ await _node(async_session, repo.id, "src/weirdly.py")
+ await _node(async_session, repo.id, "src/uncovered.py")
+ await _node(async_session, repo.id, "tests/test_misc.py", is_test=True)
+ await save_test_coverage(
+ async_session,
+ repo.id,
+ [
+ TestCoverage(
+ test_id="tests/test_misc.py::test_it",
+ file_path="src/weirdly.py",
+ covered_lines=[1, 2],
+ source_format="coverage.py",
+ test_file="tests/test_misc.py",
+ )
+ ],
+ source_format="coverage.py",
+ )
+ await async_session.commit()
+
+ analyzer = PRBlastRadiusAnalyzer(async_session, repo.id)
+ gaps = await analyzer._find_test_gaps(["src/weirdly.py", "src/uncovered.py"])
+
+ # weirdly.py: coverage-proven tested -> not a gap.
+ assert "src/weirdly.py" not in gaps
+ # uncovered.py: no coverage row AND no name-matched test -> still a gap.
+ assert "src/uncovered.py" in gaps
+
+
+async def test_filename_fallback_when_map_has_no_data(async_session):
+ repo = await insert_repo(async_session)
+ # No coverage rows at all -> pure filename fallback (pre-Phase-3 behavior):
+ # foo.py has a name-matched test, bar.py does not.
+ await _node(async_session, repo.id, "src/foo.py")
+ await _node(async_session, repo.id, "src/bar.py")
+ await _node(async_session, repo.id, "tests/test_foo.py", is_test=True)
+ await async_session.commit()
+
+ analyzer = PRBlastRadiusAnalyzer(async_session, repo.id)
+ gaps = await analyzer._find_test_gaps(["src/foo.py", "src/bar.py"])
+
+ assert "src/foo.py" not in gaps # name-matched test -> not a gap
+ assert "src/bar.py" in gaps # no test, no coverage -> gap