Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(graph): keep test-to-test edges out of PageRank and betweenness (#2695)

Swati Ahujacab9dea9d48d
diff --git a/packages/core/src/repowise/core/ingestion/graph/_centrality_cache.py b/packages/core/src/repowise/core/ingestion/graph/_centrality_cache.py
index aca79d883..ffed3ecde 100644
--- a/packages/core/src/repowise/core/ingestion/graph/_centrality_cache.py
+++ b/packages/core/src/repowise/core/ingestion/graph/_centrality_cache.py
@@ -49,7 +49,9 @@ log = structlog.get_logger(__name__)
# dependent values and two installs at the same commit would still disagree.
# 3 when entries gained the scored edge set and commit, needed to answer "how
# far has the graph drifted, and which nodes were never in this scoring".
-_CACHE_VERSION = 3
+# 4 when file betweenness moved to the graph without test-to-test edges; a
+# churn-budget hit would otherwise keep serving the old scoring.
+_CACHE_VERSION = 4
_CACHE_FILENAME = "centrality_cache.pkl"
__all__ = ["BetweennessScoring", "CentralityCache", "subgraph_signature"]
diff --git a/packages/core/src/repowise/core/ingestion/graph/_metrics.py b/packages/core/src/repowise/core/ingestion/graph/_metrics.py
index f41b4ec0f..527b9fd7f 100644
--- a/packages/core/src/repowise/core/ingestion/graph/_metrics.py
+++ b/packages/core/src/repowise/core/ingestion/graph/_metrics.py
@@ -14,6 +14,7 @@ snapshot. The structural graph stays available for traversal.
from __future__ import annotations
+from collections.abc import Callable
from typing import Any
import networkx as nx
@@ -38,6 +39,11 @@ _LARGE_REPO_THRESHOLD = 30_000 # nodes — above this, algorithms are expensive
_CHURN_BUDGET_FRACTION = 0.002
+def is_test_to_test_edge(g: nx.DiGraph, u: str, v: str) -> bool:
+ """Whether both ends are test material; such edges carry no centrality."""
+ return bool(g.nodes[u].get("is_test") and g.nodes[v].get("is_test"))
+
+
class MetricsMixin:
"""Centrality + community + degree metrics for :class:`GraphBuilder`."""
@@ -143,18 +149,43 @@ class MetricsMixin:
repo a second full copy of the file graph is real memory for a graph we
only ever read. Callers must treat the result as read-only.
"""
- cached = self._cycle_subgraph_cache
+ return self._file_edge_view(
+ "_cycle_subgraph_cache", lambda base, u, v, d: is_cohesion_edge(d)
+ )
+
+ def centrality_subgraph(self) -> nx.DiGraph:
+ """Return :meth:`file_subgraph` minus test-to-test edges, for PageRank and betweenness.
+
+ A suite's imports of its own fixtures and helpers say nothing about
+ which production files matter, yet they made ``tests/conftest.py`` rank
+ beside core modules. An edge with one production endpoint stays: a test
+ importing a module is a real (weak) vote for it.
+
+ Degrees and dead code keep :meth:`file_subgraph`: a helper only tests
+ import is still reachable. Callers must treat the result as read-only.
+ """
+ return self._file_edge_view(
+ "_centrality_subgraph_cache",
+ lambda base, u, v, d: is_test_to_test_edge(base, u, v),
+ )
+
+ def _file_edge_view(
+ self, cache_attr: str, hide: Callable[[nx.DiGraph, str, str, dict], bool]
+ ) -> nx.DiGraph:
+ """Cache a ``restricted_view`` of :meth:`file_subgraph` hiding edges where *hide* holds."""
+ cached = getattr(self, cache_attr)
if cached is not None:
return cached
# Resolve the base graph BEFORE taking the lock: file_subgraph() takes
# the same lock, and threading.Lock is not reentrant.
base = self.file_subgraph()
with self._subgraph_lock:
- if self._cycle_subgraph_cache is not None:
- return self._cycle_subgraph_cache
- cohesion = [(u, v) for u, v, d in base.edges(data=True) if is_cohesion_edge(d)]
- sub = nx.restricted_view(base, [], cohesion)
- self._cycle_subgraph_cache = sub
+ cached = getattr(self, cache_attr)
+ if cached is not None:
+ return cached
+ hidden = [(u, v) for u, v, d in base.edges(data=True) if hide(base, u, v, d)]
+ sub = nx.restricted_view(base, [], hidden)
+ setattr(self, cache_attr, sub)
return sub
def symbol_subgraph(self) -> nx.DiGraph:
@@ -215,7 +246,7 @@ class MetricsMixin:
"""Return PageRank scores for file nodes only (cached)."""
if self._pagerank_cache is not None:
return self._pagerank_cache
- filtered = self.file_subgraph()
+ filtered = self.centrality_subgraph()
if filtered.number_of_nodes() == 0:
self._pagerank_cache = {}
return self._pagerank_cache
@@ -232,7 +263,7 @@ class MetricsMixin:
"""Return betweenness centrality for file nodes (cached)."""
if self._betweenness_cache is not None:
return self._betweenness_cache
- g = self.file_subgraph()
+ g = self.centrality_subgraph()
if g.number_of_nodes() == 0:
self._betweenness_cache = {}
return self._betweenness_cache
diff --git a/packages/core/src/repowise/core/ingestion/graph/builder.py b/packages/core/src/repowise/core/ingestion/graph/builder.py
index 1bc35dcf9..48e3ffde2 100644
--- a/packages/core/src/repowise/core/ingestion/graph/builder.py
+++ b/packages/core/src/repowise/core/ingestion/graph/builder.py
@@ -120,6 +120,7 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
self._file_subgraph_cache: nx.DiGraph | None = None
self._symbol_subgraph_cache: nx.DiGraph | None = None
self._cycle_subgraph_cache: nx.DiGraph | None = None
+ self._centrality_subgraph_cache: nx.DiGraph | None = None
# Shared import-name maps (built once per build(), injected into the
# call + heritage resolvers; reset whenever files change).
self._import_name_maps: Any | None = None
@@ -134,6 +135,9 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
# ``__setstate__`` recreates the lock.
state = self.__dict__.copy()
state["_subgraph_lock"] = None
+ # restricted_view holds lambdas, which don't pickle; they rebuild lazily.
+ state["_cycle_subgraph_cache"] = None
+ state["_centrality_subgraph_cache"] = None
return state
def __setstate__(self, state: dict) -> None:
@@ -145,6 +149,7 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
# this is an explicit cross-version process boundary — default it rather
# than let the first cycle_subgraph() call raise AttributeError.
self.__dict__.setdefault("_cycle_subgraph_cache", None)
+ self.__dict__.setdefault("_centrality_subgraph_cache", None)
def set_tsconfig_resolver(self, resolver: Any) -> None:
"""Attach a :class:`TsconfigResolver` for TS/JS path-alias resolution."""
@@ -166,6 +171,7 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
self._file_subgraph_cache = None
self._symbol_subgraph_cache = None
self._cycle_subgraph_cache = None
+ self._centrality_subgraph_cache = None
self._import_name_maps = None
def _invalidate_subgraph_caches(self) -> None:
@@ -179,6 +185,7 @@ class GraphBuilder(MetricsMixin, ResolveMixin, EdgesMixin, SerializeMixin, Rehyd
self._file_subgraph_cache = None
self._symbol_subgraph_cache = None
self._cycle_subgraph_cache = None
+ self._centrality_subgraph_cache = None
def release_graph(self) -> None:
"""Drop the in-memory NetworkX object after metrics are materialized.
diff --git a/packages/core/src/repowise/core/persistence/stores/in_process_graph_store.py b/packages/core/src/repowise/core/persistence/stores/in_process_graph_store.py
index eed2e16ae..a4773a020 100644
--- a/packages/core/src/repowise/core/persistence/stores/in_process_graph_store.py
+++ b/packages/core/src/repowise/core/persistence/stores/in_process_graph_store.py
@@ -119,7 +119,7 @@ class InProcessGraphStore(GraphStore):
def pagerank(self) -> dict[str, float]:
if self._pagerank_cache is None:
- file_subgraph = self._file_subgraph()
+ file_subgraph = self._centrality_subgraph()
if file_subgraph.number_of_nodes() == 0:
self._pagerank_cache = {}
else:
@@ -128,7 +128,7 @@ class InProcessGraphStore(GraphStore):
def betweenness_centrality(self) -> dict[str, float]:
if self._betweenness_cache is None:
- file_subgraph = self._file_subgraph()
+ file_subgraph = self._centrality_subgraph()
if file_subgraph.number_of_nodes() == 0:
self._betweenness_cache = {}
else:
@@ -182,7 +182,7 @@ class InProcessGraphStore(GraphStore):
for one graph.
That parity covers :meth:`pagerank` and :meth:`betweenness_centrality`,
- the two builder metrics computed over ``file_subgraph``. It does *not*
+ which both sides compute over it minus test-to-test edges. It does *not*
make :meth:`communities` agree with the pipeline: that path runs
``detect_file_communities``, which excludes ``external`` deliberately,
allowlists ``FILE_DEPENDENCY_EDGE_TYPES`` rather than denying temporal
@@ -212,5 +212,13 @@ class InProcessGraphStore(GraphStore):
)
return sub
+ def _centrality_subgraph(self) -> nx.DiGraph:
+ """:meth:`_file_subgraph` minus test-to-test edges, as ``GraphBuilder.centrality_subgraph``."""
+ from ...ingestion.graph._metrics import is_test_to_test_edge
+
+ sub = self._file_subgraph()
+ sub.remove_edges_from([(u, v) for u, v in sub.edges() if is_test_to_test_edge(sub, u, v)])
+ return sub
+
__all__ = ["InProcessGraphStore"]
diff --git a/tests/unit/ingestion/test_centrality_cache.py b/tests/unit/ingestion/test_centrality_cache.py
index 1b0025b5a..3bfcbc637 100644
--- a/tests/unit/ingestion/test_centrality_cache.py
+++ b/tests/unit/ingestion/test_centrality_cache.py
@@ -186,7 +186,7 @@ def test_stale_cache_version_degrades_to_recompute(tmp_path):
domain="centrality_cache.pkl",
)
assert _lookup(CentralityCache(tmp_path), "symbol", _graph([("a", "b")]), "sig-1") is None
- assert cc._CACHE_VERSION == 3
+ assert cc._CACHE_VERSION == 4
async def test_compute_metrics_parallel_with_cache(tmp_path):
diff --git a/tests/unit/ingestion/test_centrality_subgraph.py b/tests/unit/ingestion/test_centrality_subgraph.py
new file mode 100644
index 000000000..8bc9e353a
--- /dev/null
+++ b/tests/unit/ingestion/test_centrality_subgraph.py
@@ -0,0 +1,113 @@
+"""PageRank and betweenness ignore test-to-test edges; degrees keep them."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+import pytest
+
+from repowise.core.ingestion import GraphBuilder
+from repowise.core.ingestion.models import FileInfo
+from repowise.core.ingestion.parser import parse_file
+from repowise.core.test_paths import is_test_related_path
+
+_SOURCES = {
+ "app/__init__.py": "",
+ "app/util.py": "def clamp(x):\n return x\n",
+ "app/core.py": "from app.util import clamp\n\ndef run(x):\n return clamp(x)\n",
+ "app/api.py": "from app.core import run\n\ndef serve():\n return run(1)\n",
+ "app/cli.py": "from app.api import serve\n\ndef main():\n return serve()\n",
+}
+# Every test pulls fixtures from conftest, so it collects many in-edges.
+_TESTS = {
+ "tests/__init__.py": "",
+ "tests/conftest.py": "from app.core import run\n\ndef client():\n return run\n",
+ **{
+ f"tests/test_{name}.py": (
+ f"from tests.conftest import client\nfrom app.{name} import *\n\n"
+ f"def test_{name}():\n assert client()\n"
+ )
+ for name in ("util", "core", "api", "cli")
+ },
+}
+
+
+def _parsed(root: Path, rel: str, content: str):
+ abs_ = root / rel
+ abs_.parent.mkdir(parents=True, exist_ok=True)
+ abs_.write_text(content)
+ fi = FileInfo(
+ path=rel,
+ abs_path=str(abs_),
+ language="python", # type: ignore[arg-type]
+ size_bytes=abs_.stat().st_size,
+ git_hash="",
+ last_modified=datetime.now(),
+ is_test=is_test_related_path(rel, "python"),
+ is_config=False,
+ is_api_contract=False,
+ is_entry_point=False,
+ )
+ return parse_file(fi, content.encode("utf-8"))
+
+
+def pytest_conftest_hub(tmp_path: Path) -> GraphBuilder:
+ gb = GraphBuilder(str(tmp_path))
+ for rel, content in {**_SOURCES, **_TESTS}.items():
+ gb.add_file(_parsed(tmp_path, rel, content))
+ gb.build()
+ return gb
+
+
+def test_conftest_ranks_below_every_source_module(pytest_conftest_hub: GraphBuilder) -> None:
+ gb = pytest_conftest_hub
+ # The fixture only proves something if conftest really is a test-import hub.
+ assert gb.file_subgraph().in_degree("tests/conftest.py") == 4
+
+ pr = gb.pagerank()
+ sources = [p for p in _SOURCES if p != "app/__init__.py"]
+ assert pr["tests/conftest.py"] < min(pr[p] for p in sources)
+
+
+def test_only_edges_between_two_tests_are_hidden(pytest_conftest_hub: GraphBuilder) -> None:
+ gb = pytest_conftest_hub
+ view = gb.centrality_subgraph()
+ assert not view.has_edge("tests/test_core.py", "tests/conftest.py")
+ assert view.has_edge("tests/test_core.py", "app/core.py")
+ assert view.has_edge("tests/conftest.py", "app/core.py")
+ assert set(view.nodes) == set(gb.file_subgraph().nodes)
+
+
+def test_degrees_and_file_subgraph_keep_test_edges(pytest_conftest_hub: GraphBuilder) -> None:
+ gb = pytest_conftest_hub
+ assert gb.file_subgraph().has_edge("tests/test_core.py", "tests/conftest.py")
+ assert gb.in_degree()["tests/conftest.py"] == 4
+
+
+def test_view_is_cached_and_reset_with_the_graph(pytest_conftest_hub: GraphBuilder) -> None:
+ gb = pytest_conftest_hub
+ view = gb.centrality_subgraph()
+ assert gb.centrality_subgraph() is view
+ gb._invalidate_subgraph_caches()
+ assert gb.centrality_subgraph() is not view
+
+
+def test_builder_with_built_views_still_pickles(pytest_conftest_hub: GraphBuilder) -> None:
+ import pickle
+
+ gb = pytest_conftest_hub
+ gb.centrality_subgraph()
+ gb.cycle_subgraph()
+ reloaded = pickle.loads(pickle.dumps(gb))
+ assert reloaded.pagerank() == gb.pagerank()
+ assert set(reloaded.centrality_subgraph().edges) == set(gb.centrality_subgraph().edges)
+
+
+def test_graph_store_scores_the_same_view(pytest_conftest_hub: GraphBuilder) -> None:
+ from repowise.core.persistence.stores import InProcessGraphStore
+
+ gb = pytest_conftest_hub
+ store = InProcessGraphStore.from_graph(gb.graph())
+ assert store.pagerank() == pytest.approx(gb.pagerank())