Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(health): unblock the refactoring surface, spread plan ranking, drop unextractable clone plans (#1413)
Raghav Chamadiya7b6f785cfb97
diff --git a/packages/core/src/repowise/core/analysis/health/refactoring/extract_helper.py b/packages/core/src/repowise/core/analysis/health/refactoring/extract_helper.pyindex 8fdced152..4b2c28bb0 100644--- a/packages/core/src/repowise/core/analysis/health/refactoring/extract_helper.py+++ b/packages/core/src/repowise/core/analysis/health/refactoring/extract_helper.py@@ -38,6 +38,7 @@ still worth extracting but ranks ``medium``.from __future__ import annotations+import refrom typing import Anyfrom ....test_paths import is_test_related_path@@ -69,6 +70,131 @@ _REGION_SLACK = 1_MAX_SNIPPET_LINES = 40+# Line shapes that declare something rather than do something. A clone made+# entirely of these is not extractable: you cannot lift an import block, a+# parameter list or a run of dataclass fields into a shared helper, because+# there is no behaviour there to share — only a shape that happens to repeat.+_IMPORT_PREFIXES = (+ "import ",+ "from ",+ "using ",+ "#include",+ "export ",+ "require(",+ "const {",+ "@import",+)+# A constant RHS. ``name = compute(x)`` is real code and must stay a candidate;+# ``name = []`` / ``name = "x"`` / ``FOO = 3`` is a declaration.+_CONST_RHS = re.compile(+ r"""^\s*(+ None|True|False|null|true|false # literals+ | -?\d[\d_.eE+-]* # numbers+ | ["'].*["'] # strings+ | \[\s*\] | \{\s*\} | \(\s*\) # empty collections+ | (list|dict|set|tuple|str|int|float|bool)\s*\(\s*\)+ )\s*,?\s*$""",+ re.VERBOSE,+)+# The field-declaration constructs a dataclass-style default is allowed to use.+# Deliberately a closed vocabulary rather than "any call": ``items: list[str] =+# field(default_factory=list)`` declares a field, but ``result: int =+# calculate_total(items, rate)`` is behaviour that merely carries an annotation,+# and accepting arbitrary right-hand sides here silently dropped the second —+# a false negative on one of the most common shapes in typed Python and TS.+_FIELD_CTOR = re.compile(+ r"^\s*(dataclasses\.|attr\.|attrs\.|pydantic\.|msgspec\.)?"+ r"(field|Field|ib|attrib|mapped_column|Column|relationship)\s*\(",+)+_DECL_LINE = re.compile(+ r"""^\s*(+ [\w.\[\]"']+ \s* : \s* [^=]+ # annotated, no default: name: type+ | [\w]+ \s* ,? # bare parameter / enum member+ | [)\]}]\s* [:,]? \s* (->.*)? :? \s* # a closer, with or without a return type+ | (async\s+)?(def|function|fn|func|class|interface|type|struct)\s+[\w<>]+\s*[(<{:]?\s*+ | @[\w.]+ \s* \(? # decorator / annotation+ )$""",+ re.VERBOSE,+)+_CONTROL_FLOW = re.compile(+ r"\b(if|else|elif|for|while|try|except|catch|finally|return|yield|raise|throw|"+ r"with|switch|case|await|match|break|continue|next|fallthrough|defer|goto|redo|retry)\b"+)+++def _code_lines(block: list[str]) -> list[str]:+ """*block* minus blanks and whole-line comments.++ A leading ``*`` is a JSDoc/javadoc comment continuation (``* @param x``) —+ but it is also a C/Go pointer dereference (``*out = compute(a, b);``), and+ stripping those left an empty line list, which the caller reads as "nothing+ but declarations". So a ``*`` line only counts as a comment when it carries+ no assignment and no statement terminator.+ """+ out = []+ for raw in block:+ s = raw.strip()+ if not s:+ continue+ if s.startswith(("#", "//", "/*", "*/", '"""', "'''", "<!--")):+ continue+ if s.startswith("*") and "=" not in s and ";" not in s:+ continue+ out.append(s)+ return out+++def _is_declaration_only(block: list[str]) -> bool:+ """True when every line of *block* declares rather than executes.++ Three real false positives from this repo's own index motivated this, all+ of them plans proposing to extract something no editor could extract:++ - a 21-line ``from repowise.cli.helpers import (...)`` block, shared with+ ``routers/workspace.py`` because both import the same names;+ - two 20-line ``def update_command(...)`` / ``def run_update(...)``+ *signatures*, paired because their parameter lists agree;+ - runs of ``field(default_factory=list)`` dataclass members across four+ unrelated modules (the case filed as C6, which arrived at the top of the+ payload carrying the highest occurrence count and blast radius, i.e. the+ most-trusted slot).++ A single line with control flow, or an unannotated assignment whose RHS is+ not a constant, is enough to keep the block a candidate — the gate has to+ stay conservative, because a false *negative* here silently drops real+ duplication, which is the more expensive mistake.++ Deliberately *not* applied in the duplication detector or the+ ``dry_violation`` biomarker: those feed calibrated scores that are frozen,+ and the defect here is the advice, not the measurement. The block still+ counts as duplication; it just no longer produces a plan to extract it.+ """+ lines = _code_lines(block)+ if not lines:+ return True+ for line in lines:+ if _CONTROL_FLOW.search(line):+ return False+ if line.startswith(_IMPORT_PREFIXES) or line in ("(", ")", "):", "}", "];", "{"):+ continue+ if _DECL_LINE.match(line):+ continue+ # An assignment is a declaration only by what it assigns. A constant+ # RHS is inert either way; an annotated line may additionally use a+ # field-declaration constructor (``field(default_factory=list)``).+ # Anything else computed is behaviour and stays extractable, annotated+ # or not — ``result: int = calculate_total(items, rate)`` is code that+ # happens to carry a type, not a declaration.+ head, sep, rhs = line.partition("=")+ if sep:+ if _CONST_RHS.match(rhs):+ continue+ if ":" in head and _FIELD_CTOR.match(rhs):+ continue+ return False+ return True++def _is_generated_path(path: str) -> bool:"""Non-refactorable generated/append-only code (DB migrations, vendoredbundles). Their boilerplate duplicates heavily but extracting a shared@@ -228,6 +354,17 @@ class ExtractHelperDetector(RefactoringDetector):((s, e) for f, s, e in occurrences if f == ctx.file_path),(block.anchor_start, block.anchor_end),)++ # Reject blocks with nothing to extract (imports, parameter lists, field+ # runs). Read off the anchor region, the one occurrence whose source this+ # pass holds; the block is identical across sites by definition, so one+ # read decides it for all of them. No source threaded (non-clone file, an+ # unreadable one) means no evidence to reject on, and the gate abstains+ # rather than guessing.+ region_lines = self._region_lines(ctx, anchor_region)+ if region_lines and _is_declaration_only(region_lines):+ return None+impact = self._impact_for_block(anchor_region, impact_lookup)is_intra = len(occ_files) == 1@@ -308,6 +445,20 @@ class ExtractHelperDetector(RefactoringDetector):"""return {"directory": _common_directory(occ_files)}+ @staticmethod+ def _region_lines(ctx: RefactoringContext, region: tuple[int, int]) -> list[str]:+ """The anchor file's source for *region*, or ``[]`` when none is+ threaded. Clone ranges are 1-indexed and inclusive; the range is clamped+ to the file because a region can outlive the read that produced it."""+ lines = ctx.source_lines+ if not lines:+ return []+ lo = max(region[0], 1)+ hi = min(region[1], len(lines))+ if hi < lo:+ return []+ return list(lines[lo - 1 : hi])+@staticmethoddef _snippet_for(ctx: RefactoringContext, anchor_region: tuple[int, int]@@ -321,16 +472,10 @@ class ExtractHelperDetector(RefactoringDetector):``(None, None, False)`` when no source was threaded (non-clone file, oran unreadable one), leaving the plan on its line ranges alone."""- lines = ctx.source_lines- if not lines:- return None, None, False- start, end = anchor_region- # Clamp to the file; clone ranges are 1-indexed and inclusive.- lo = max(start, 1)- hi = min(end, len(lines))- if hi < lo:+ block = ExtractHelperDetector._region_lines(ctx, anchor_region)+ if not block:return None, None, False- block = lines[lo - 1 : hi]+ lo = max(anchor_region[0], 1)truncated = len(block) > _MAX_SNIPPET_LINESif truncated:block = block[:_MAX_SNIPPET_LINES]diff --git a/packages/server/src/repowise/server/mcp_server/tool_health.py b/packages/server/src/repowise/server/mcp_server/tool_health.pyindex 69c311af9..981886de1 100644--- a/packages/server/src/repowise/server/mcp_server/tool_health.py+++ b/packages/server/src/repowise/server/mcp_server/tool_health.py@@ -338,7 +338,15 @@ def _directive("recovers_points": recovers,"share_of_repo_gap_pct": (round(100.0 * recovers / gap_points, 1) if gap_points else None),"then": [m.file_path for m in by_leverage[1:3]],- "plan_via": "get_health(include=['refactoring'])",+ # Projected, not bare. ``include`` adds a block without subtracting the+ # dashboard, and five ranked lists at the default ``limit`` compose: the+ # bare ``include=['refactoring']`` measured 70,776 chars on this repo+ # (refactoring_plans 34%, the other four lists 59%) and simply fails the+ # MCP token cap, so the one call the directive tells an agent to make was+ # the one call it could not complete. ``only`` already exists to fix this+ # (it gates the work as well as the payload); the directive just has to+ # ask for it. Same block, ~24k chars, no dashboard restated.+ "plan_via": "get_health(include=['refactoring'], only=['refactoring_plans'])","plan_addresses_reason": addresses,}# Only speak when there is a named cause to speak about. With no lead the@@ -1167,12 +1175,31 @@ async def get_health(key=lambda r: (deficit_by_path.get(r.file_path, 0), r.impact_delta or 0.0),reverse=True,)+ # ...then spread the cap across files. Deficit is a *file* property, so a+ # pure deficit sort puts every plan on the worst file ahead of every plan+ # on the second worst: asking for the top 8 returned 8 plans on one file+ # out of 1,903, and an agent asking "what should I refactor?" saw no view+ # of the repo at all. Round-robin one plan per file per pass, files in+ # their ranked order (a file ranks by its best plan) and plans in theirs,+ # so the head spans as many files as it has rows. The order is otherwise+ # unchanged — the worst file still leads, it just no longer owns the list.+ by_file: dict[str, list[Any]] = {}+ for r in ranked:+ by_file.setdefault(r.file_path, []).append(r)+ spread: list[Any] = []+ while len(spread) < limit and by_file:+ for path in list(by_file):+ spread.append(by_file[path].pop(0))+ if not by_file[path]:+ del by_file[path]+ if len(spread) >= limit:+ breakresult["refactoring_plans"] = [{**_serialize_refactoring(r),"file_weighted_deficit": deficit_by_path.get(r.file_path, 0),}- for r in ranked[:limit]+ for r in spread]result["refactoring_plans_total"] = len(refactoring_rows)# The deterministic prose suggestion is the fallback for biomarkersdiff --git a/tests/unit/health/test_refactoring_extract_helper.py b/tests/unit/health/test_refactoring_extract_helper.pyindex a0258245f..c3a73ca95 100644--- a/tests/unit/health/test_refactoring_extract_helper.py+++ b/tests/unit/health/test_refactoring_extract_helper.py@@ -507,3 +507,173 @@ def test_deterministic_and_stable_order():# Bigger recovered impact first.assert [s.line_start for s in a] == [100, 10]assert [(s.target_symbol, s.plan) for s in a] == [(s.target_symbol, s.plan) for s in b]+++# ---- declaration-only blocks are not extractable -------------------------+#+# A clone made entirely of declarations has no behaviour to share, so a plan+# proposing to extract it is advice no editor can follow. Three real shapes+# from this repo's own index motivated the gate, and the worst of them arrived+# in the most-trusted slot in the payload (highest occurrence count, biggest+# blast radius): an `x49` import block and an `x32` one.+#+# The unit cases below cannot be revert-tested (the helper is new, so a revert+# cannot import it); the `detect_refactorings` cases can, and do — without the+# gate the detector emits a plan for every one of them.+++def _decl_source(body: list[str], *, at: int, total: int = 80) -> list[str]:+ """A file whose lines *at*..*at+len(body)* are *body*, rest inert code."""+ lines = [f" value_{i} = compute_{i}(seed)" for i in range(1, total + 1)]+ lines[at - 1 : at - 1 + len(body)] = body+ return lines+++_IMPORT_BLOCK = [+ "from repowise.cli.helpers import (",+ " clear_update_queued,",+ " console,",+ " consume_update_pending,",+ " ensure_repowise_dir,",+ " find_workspace_root,",+ " get_head_commit,",+ " load_config,",+ " save_state,",+ ")",+]++_SIGNATURE_BLOCK = [+ "def update_command(",+ " path: str | None,",+ " provider_name: str | None,",+ " since: str | None,",+ " dry_run: bool,",+ " workspace: bool,",+ " index_only: bool = False,",+ " concurrency: int = 10,",+ ")",+]++_FIELD_BLOCK = [+ " name: str = ''",+ " items: list[str] = field(default_factory=list)",+ " mapping: dict[str, int] = field(default_factory=dict)",+ " alternatives: list[str] = field(default_factory=list)",+ " count: int = 0",+ " enabled: bool = False",+ " label: str = 'none'",+ " weight: float = 1.0",+]++_REAL_BLOCK = [+ " if result is not None:",+ " try:",+ " save_knowledge_graph_json(repo_path, result)",+ " except Exception as exc:",+ " degraded.append(exc)",+ " emitter.done(ok=True, pages_generated=0)",+ " consume_update_pending(repo_path, head)",+ " total = compute_total(rows)",+ " scaled = total * weight",+]+++def _plans_for(body: list[str]):+ """Plans the detector emits for a cross-file clone whose block is *body*."""+ pair = _pair("pkg/a.py", "pkg/b.py", 10, 10 + len(body) - 1, 40, 40 + len(body) - 1)+ return [+ s+ for s in detect_refactorings(+ _ctx("pkg/a.py", [pair], source_lines=_decl_source(body, at=10))+ )+ if s.refactoring_type == "extract_helper"+ ]+++def test_import_block_clone_emits_no_plan():+ """You cannot extract `from x import (...)` into a shared helper."""+ assert _plans_for(_IMPORT_BLOCK) == []+++def test_function_signature_clone_emits_no_plan():+ """Two functions whose parameter lists agree are not duplicated behaviour."""+ assert _plans_for(_SIGNATURE_BLOCK) == []+++def test_dataclass_field_run_emits_no_plan():+ """Unrelated dataclasses sharing a run of `field(default_factory=...)`+ declarations share a shape, not code. This is the case filed as C6."""+ assert _plans_for(_FIELD_BLOCK) == []+++def test_real_duplicated_code_still_emits_a_plan():+ """The gate must stay conservative: a false negative silently drops real+ duplication, which is the more expensive mistake."""+ plans = _plans_for(_REAL_BLOCK)+ assert len(plans) == 1+ assert plans[0].plan["duplicated_lines"] == len(_REAL_BLOCK)+++def test_gate_abstains_when_no_source_is_threaded():+ """No source means no evidence to reject on, so the plan survives — the+ detector's pre-existing behaviour for non-clone / unreadable files."""+ pair = _pair("pkg/a.py", "pkg/b.py", 10, 25, 40, 55)+ plans = [+ s+ for s in detect_refactorings(_ctx("pkg/a.py", [pair], source_lines=None))+ if s.refactoring_type == "extract_helper"+ ]+ assert len(plans) == 1+++def test_declaration_only_unit_cases():+ from repowise.core.analysis.health.refactoring.extract_helper import (+ _is_declaration_only,+ )++ # Rejected: nothing to extract.+ assert _is_declaration_only(_IMPORT_BLOCK)+ assert _is_declaration_only(_SIGNATURE_BLOCK)+ assert _is_declaration_only(_FIELD_BLOCK)+ assert _is_declaration_only([" ALPHA = 'a'", " BETA = 'b'", " GAMMA = 'c'"])+ assert _is_declaration_only(["import os", "import sys", "from x import y"])+ assert _is_declaration_only(['export { A } from "./a";', 'export * from "./b";'])+ assert _is_declaration_only([" # only a comment", "", " # another"])++ # Kept: real behaviour, by three different routes.+ assert not _is_declaration_only([" if x:", " go()"]) # control flow+ assert not _is_declaration_only([" emit(a)", " emit(b)"]) # statement calls+ assert not _is_declaration_only([" t = compute(x)"]) # computed assignment+ assert not _is_declaration_only([" total = sum_rows(rows)", " n = len(total)"])++ # A type annotation does not make code a declaration. This is the shape an+ # adversarial review caught the first version of the gate dropping, and it+ # is one of the most common lines in typed Python and TypeScript, so it is+ # the highest-frequency false negative the gate could have had.+ assert not _is_declaration_only(+ [+ " result: int = calculate_total(items, tax_rate)",+ " status: str = fetch_status(connection)",+ ]+ )+ # What separates it from a field run is the *constructor*, not the call: a+ # closed vocabulary of field declarations is still a declaration.+ assert _is_declaration_only(+ [+ " items: list[str] = field(default_factory=list)",+ " id: int = Column(Integer)",+ " kids: list = relationship('K')",+ ]+ )++ # A leading `*` is a javadoc continuation in one language and a pointer+ # dereference in another. Stripping it unconditionally emptied the line list+ # and read as "nothing but declarations", defanging the gate for C and Go.+ assert not _is_declaration_only(["*x = compute_value(a, b);", "*y = another(c);"])+ assert _is_declaration_only([" * @param foo the thing", " * @returns nothing"])++ # Single-word control flow carries no other token to recognise it by, so it+ # has to be in the keyword list or it reads as a bare identifier.+ assert not _is_declaration_only([" continue", " continue"])+ assert not _is_declaration_only([" break", " break"])+ assert not _is_declaration_only([" defer f.Close()", " defer g.Close()"])diff --git a/tests/unit/server/mcp/test_health.py b/tests/unit/server/mcp/test_health.pyindex 32dad5af9..affda8418 100644--- a/tests/unit/server/mcp/test_health.py+++ b/tests/unit/server/mcp/test_health.py@@ -1063,3 +1063,94 @@ async def test_coverage_payload_shape_is_unchanged(setup_mcp, health_data):targeted = await get_health(include=["coverage"], targets=["src/auth/service.py"])for row in targeted["coverage"]["files"]:assert "covered_lines" in row+++async def test_refactoring_plans_spread_across_files(setup_mcp, health_data, session):+ """The cap is spread over files instead of being spent on the worst one.++ ``deficit_by_path`` is a *file* property, so a pure deficit sort puts every+ plan on the worst file ahead of every plan on the second worst. Measured on+ this repo's own index before the fix, asking for the top 8 plans returned 8+ plans on a single file out of 1,903 — an agent asking "what should I+ refactor?" got no view of the repo at all. Filed as C4.+ """+ from repowise.server.mcp_server import get_health++ # Six plans on the worst file, two on the other. Impact descends within the+ # worst file so a deficit-then-impact sort would take all six of them first.+ await _seed_plans(+ session,+ health_data,+ [+ {+ "file_path": "src/auth/service.py",+ "target_symbol": f"worst_{i}",+ "source_biomarker": "complex_method",+ "impact_delta": 9.0 - i,+ }+ for i in range(6)+ ]+ + [+ {+ "file_path": "src/utils/helpers.py",+ "target_symbol": f"other_{i}",+ "source_biomarker": "complex_method",+ "impact_delta": 0.1,+ }+ for i in range(2)+ ],+ )++ plans = (await get_health(include=["refactoring"], limit=4))["refactoring_plans"]+ assert len(plans) == 4+ # Both files are represented rather than the worst file owning the list.+ assert len({p["file_path"] for p in plans}) == 2+ # The worst file still leads — spreading reorders within the cap, it does+ # not demote the file the directive names.+ assert plans[0]["file_path"] == "src/auth/service.py"+ # Within a file, the higher-impact plan still comes first.+ worst = [p["target_symbol"] for p in plans if p["file_path"] == "src/auth/service.py"]+ assert worst == sorted(worst, key=lambda t: int(t.split("_")[1]))+++async def test_refactoring_spread_is_exhaustive_when_one_file_has_them_all(+ setup_mcp, health_data, session+):+ """One file holding every plan must still fill the cap — the round-robin+ drains a file's queue rather than capping it at one row per file."""+ from repowise.server.mcp_server import get_health++ await _seed_plans(+ session,+ health_data,+ [+ {+ "file_path": "src/auth/service.py",+ "target_symbol": f"only_{i}",+ "source_biomarker": "complex_method",+ "impact_delta": 5.0 - i,+ }+ for i in range(5)+ ],+ )++ plans = (await get_health(include=["refactoring"], limit=4))["refactoring_plans"]+ assert len(plans) == 4+ assert {p["file_path"] for p in plans} == {"src/auth/service.py"}+++async def test_directive_plan_via_is_projected(setup_mcp, health_data):+ """``plan_via`` must name a call that can actually complete.++ The bare ``include=['refactoring']`` measured 70,776 chars on this repo and+ fails the MCP token cap: five ranked lists at the default limit compose, and+ ``include`` adds a block without subtracting the dashboard. The directive+ told an agent to make the one call it could not finish.+ """+ from repowise.server.mcp_server import get_health++ directive = (await get_health(only=["directive"]))["directive"]+ assert "only=['refactoring_plans']" in directive["plan_via"]