Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(dataflow): give staged plan stages every value they read and write (#3471)

Raghav Chamadiya5b5c326f8fdf
diff --git a/docs/layers/REFACTORING.md b/docs/layers/REFACTORING.md
index 21124fed3..17c4ddb5a 100644
--- a/docs/layers/REFACTORING.md
+++ b/docs/layers/REFACTORING.md
@@ -229,10 +229,12 @@ Most plans answer a health finding, so per-path marker rules in
generated code comes back as a diff.
- Extract Method offers a span only when it can show the extraction keeps behavior
(every returned value written on every path, no state carried across loop
- iterations, no local a closure shares across the span's edge). Spans it cannot
- prove are dropped, so it under-reports by design. A value the span writes is
- returned whenever any path can carry it to a later read, including past a later
- write in an `else` arm or a `try` body that may raise first.
+ iterations, no local a closure shares across the span's edge, no write to a
+ `global` name without its declaration). A staged plan also leaves out a stage
+ whose inputs or outputs depend on a `switch` or `match` arm's write. Spans it
+ cannot prove are dropped, so it under-reports by design. A value the span writes
+ is returned whenever any path can carry it to a later read, including past a
+ later write in an `else` arm or a `try` body that may raise first.
- Extract Method skips spans too small to matter and spans that would carry the
original finding into the helper. A component whose branching is mostly in its
markup gets no extraction, and neither does a function that only fills one
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/cfg.py b/packages/core/src/repowise/core/analysis/health/dataflow/cfg.py
index 7e85a1f44..ac1723ce8 100644
--- a/packages/core/src/repowise/core/analysis/health/dataflow/cfg.py
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/cfg.py
@@ -552,6 +552,9 @@ class _CFGBuilder:
for handler in (c for c in try_node.children if c.type in self.lmap.catch_kinds):
first = len(self.blocks)
handler_entry = self._new("handler")
+ # The clause head (``except E as e:``) reads the exception type and
+ # binds its name; the dialect says which (``handler_def_use``).
+ self._record_head(handler_entry, handler)
self._protect(cur, protected, handler_entry)
handler_out = self._process_seq(
self._body_stmts(self._block_of(handler)), handler_entry
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/defuse.py b/packages/core/src/repowise/core/analysis/health/dataflow/defuse.py
index 4b52464f1..72263b263 100644
--- a/packages/core/src/repowise/core/analysis/health/dataflow/defuse.py
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/defuse.py
@@ -24,7 +24,7 @@ if TYPE_CHECKING:
from ..complexity.languages import LanguageNodeMap
from .cfg import CFG
- from .dialects.base import BaseDefUseDialect
+ from .dialects.base import BaseDefUseDialect, StatementDefUse
@dataclass(frozen=True)
@@ -66,7 +66,9 @@ class FunctionDefUse:
``blocks`` is keyed by block id; ``definitions`` is every write site ordered
by ``index`` (so ``definitions[i].index == i``); ``params`` is the parameter
- occurrences seeded at the entry block. ``captured`` is what nested
+ occurrences seeded at the entry block. ``outer`` is, per name the
+ function declares ``global`` / ``nonlocal``, the declaring lines.
+ ``captured`` is what nested
closures read and write of this function's variables (see
:class:`Captured`), kept apart from the per-block uses because a closure
runs when it is called, not where it is written; code that moves
@@ -77,6 +79,7 @@ class FunctionDefUse:
definitions: list[Definition]
params: tuple[Occurrence, ...]
captured: Captured = field(default_factory=Captured)
+ outer: dict[str, frozenset[int]] = field(default_factory=dict)
def block(self, block_id: int) -> BlockDefUse | None:
return self.blocks.get(block_id)
@@ -97,6 +100,7 @@ def compute_def_use(
blocks: dict[int, BlockDefUse] = {}
definitions: list[Definition] = []
counter = 0
+ outer: dict[str, set[int]] = {}
def _add_def(occ: Occurrence, block_id: int) -> None:
nonlocal counter
@@ -128,13 +132,29 @@ def compute_def_use(
for stmt in block.statements:
if stmt.node is None:
continue
- sdu = dialect.statement_def_use(stmt.node, lmap, head_only=stmt.is_head)
+ sdu = _statement_def_use(dialect, stmt.node, lmap, stmt.is_head)
for occ in sdu.defs:
_add_def(occ, block.id)
bdu.uses.extend(sdu.uses)
+ for occ in dialect.outer_declared(stmt.node):
+ outer.setdefault(occ.name, set()).add(stmt.start_line)
captured = Captured()
dialect.collect_captured(fn_node.child_by_field_name("body"), captured, lmap)
return FunctionDefUse(
- blocks=blocks, definitions=definitions, params=params, captured=captured
+ blocks=blocks,
+ definitions=definitions,
+ params=params,
+ captured=captured,
+ outer={name: frozenset(lines) for name, lines in outer.items()},
)
+
+
+def _statement_def_use(
+ dialect: BaseDefUseDialect, node: Node, lmap: LanguageNodeMap, is_head: bool
+) -> StatementDefUse:
+ """One recorded statement's def/use: an exception handler's head through
+ :meth:`~BaseDefUseDialect.handler_def_use`, anything else as a statement."""
+ if is_head and node.type in lmap.catch_kinds:
+ return dialect.handler_def_use(node)
+ return dialect.statement_def_use(node, lmap, head_only=is_head)
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/dialects/base.py b/packages/core/src/repowise/core/analysis/health/dataflow/dialects/base.py
index 6c9e936e8..9e6504d85 100644
--- a/packages/core/src/repowise/core/analysis/health/dataflow/dialects/base.py
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/dialects/base.py
@@ -340,7 +340,11 @@ class BaseDefUseDialect:
loops, decls, outer = self._closure_parts(node, lmap)
head_writes = self._loop_binder_scopes(loops, lmap, scopes)
for decl in decls:
- _add_declaration_scope(decl, writes, lmap, scopes)
+ if decl.type in lmap.catch_kinds: # a handler's name is its own
+ for d in self.handler_def_use(decl).defs:
+ scopes.setdefault(d.name, []).append((decl.start_point, decl.end_point))
+ else:
+ _add_declaration_scope(decl, writes, lmap, scopes)
if not self.closure_writes_outer:
for w in (*writes, *head_writes):
if w.name not in outer:
@@ -353,8 +357,9 @@ class BaseDefUseDialect:
def _closure_parts(
self, node: Node, lmap: LanguageNodeMap
) -> tuple[list[Node], list[Node], set[str]]:
- """The loops and declaration statements of nested scope *node* (deeper
- scopes not entered), and the names it declares outer."""
+ """The loops and declaration statements (exception handlers too) of
+ nested scope *node* (deeper scopes not entered), and the names it
+ declares outer."""
loops: list[Node] = []
decls: list[Node] = []
outer: set[str] = set()
@@ -367,7 +372,7 @@ class BaseDefUseDialect:
outer |= {occ.name for occ in _leaf_names(cur, self.identifier_kinds)}
elif cur.type in lmap.loop_kinds:
loops.append(cur)
- elif cur.type in lmap.local_decl_kinds:
+ elif cur.type in lmap.local_decl_kinds or cur.type in lmap.catch_kinds:
decls.append(cur)
stack.extend(cur.named_children)
return loops, decls, outer
@@ -453,6 +458,20 @@ class BaseDefUseDialect:
Default: none (a language with no override seeds no parameters)."""
return ()
+ def outer_declared(self, node: Node) -> list[Occurrence]:
+ """The names statement *node* declares as the module's or an
+ enclosing function's (Python ``global`` / ``nonlocal``); none in a
+ language without such statements."""
+ if node.type not in self.outer_decl_kinds:
+ return []
+ return _leaf_names(node, self.identifier_kinds)
+
+ def handler_def_use(self, handler: Node) -> StatementDefUse:
+ """What an exception handler's head reads and binds (the CFG records
+ it as the handler block's first statement). Default: nothing, the
+ reading every language had before handler heads were recorded."""
+ return StatementDefUse(defs=(), uses=())
+
def receiver(self, fn_node: Node, lmap: LanguageNodeMap) -> Receiver | None:
"""The instance *fn_node* runs on (:class:`Receiver`), ``NO_RECEIVER``
for a plain function, None when this language cannot tell.
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/dialects/python.py b/packages/core/src/repowise/core/analysis/health/dataflow/dialects/python.py
index 9b0f937dc..1cc56bd9d 100644
--- a/packages/core/src/repowise/core/analysis/health/dataflow/dialects/python.py
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/dialects/python.py
@@ -98,6 +98,16 @@ class PythonDefUseDialect(BaseDefUseDialect):
out.append(self._occ(name_node))
return tuple(out)
+ def handler_def_use(self, handler: Node) -> StatementDefUse:
+ """``except E as e:`` reads ``E`` and binds ``e``: the same ``value``
+ field shape as a ``with`` item. Python unbinds ``e`` when the handler
+ ends; that is not modelled, so a later read of ``e`` (an error in the
+ original too) sees this binding and a span holding it takes ``e`` in."""
+ defs: list[Occurrence] = []
+ uses: list[Occurrence] = []
+ self._with_item(handler, defs, uses)
+ return StatementDefUse(defs=tuple(defs), uses=tuple(uses))
+
def _own_receiver(self, fn_node: Node, lmap: LanguageNodeMap) -> Receiver | None:
"""A method's first parameter when it is ``self`` / ``cls`` and the
function sits in a class body; a nested function reaches ``self`` as a
@@ -179,6 +189,13 @@ class PythonDefUseDialect(BaseDefUseDialect):
if t == "with_item":
self._with_item(node, defs, uses)
return
+ if t == "except_clause": # a nested scope's handler, walked whole
+ self._with_item(node, defs, uses)
+ value = node.child_by_field_name("value")
+ for child in node.named_children:
+ if value is None or child.id != value.id:
+ self._process(child, defs, uses)
+ return
if t in self.member_access_kinds:
self.collect_reads(node, uses) # receiver is a read; member is not
return
@@ -296,10 +313,13 @@ class PythonDefUseDialect(BaseDefUseDialect):
named = node.child_by_field_name("name")
if named is not None and named.type in self.identifier_kinds:
return named
- # typed_parameter / splat: first identifier descendant.
+ # typed_parameter / splat: first identifier descendant (``**kw: Any``
+ # nests the splat in the typed parameter).
for child in node.named_children:
if child.type in self.identifier_kinds:
return child
+ if child.type in _SPLAT_TARGETS:
+ return self._param_name_node(child)
return None
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/dialects/ts_js.py b/packages/core/src/repowise/core/analysis/health/dataflow/dialects/ts_js.py
index 8d344e0c9..c30904dc7 100644
--- a/packages/core/src/repowise/core/analysis/health/dataflow/dialects/ts_js.py
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/dialects/ts_js.py
@@ -43,6 +43,8 @@ _SHORTHAND_PAT = "shorthand_property_identifier_pattern"
_PAIR_PATTERN = "pair_pattern"
_REST_PATTERN = "rest_pattern"
_ASSIGN_PATTERN = "assignment_pattern" # default value ``a = 1`` in a pattern
+_CATCH = "catch_clause"
+_PARAM_PATTERNS = frozenset({_ARRAY_PATTERN, _OBJECT_PATTERN, _REST_PATTERN, _ASSIGN_PATTERN})
# Member / subscript targets bind no local.
_MEMBER = "member_expression"
_SUBSCRIPT = "subscript_expression"
@@ -119,6 +121,16 @@ class TsJsDefUseDialect(BaseDefUseDialect):
self._process(node, defs, uses)
return StatementDefUse(defs=tuple(defs), uses=tuple(uses))
+ def handler_def_use(self, handler: Node) -> StatementDefUse:
+ """``catch (e)`` / ``catch ({ message })`` binds its names for the
+ handler block only."""
+ defs: list[Occurrence] = []
+ param = handler.child_by_field_name("parameter")
+ self._targets(param, defs, [])
+ if param is not None:
+ self._declare(defs, 0, param)
+ return StatementDefUse(defs=tuple(defs), uses=())
+
def declared_type(self, name: Node) -> str | None:
"""The base rule, plus ``| undefined`` for an optional parameter
(``x?: T`` holds ``T | undefined`` until something narrows it)."""
@@ -137,9 +149,13 @@ class TsJsDefUseDialect(BaseDefUseDialect):
out: list[Occurrence] = []
sink: list[Occurrence] = []
for child in params_node.named_children:
+ # TS wraps each parameter (``required_parameter``); JS lists the name
+ # or pattern itself (``b = 1``, ``{ a, b }``, ``[a]``, ``...rest``).
pattern = child.child_by_field_name("pattern")
- if pattern is None:
- pattern = child if child.type in self.identifier_kinds else None
+ if pattern is None and (
+ child.type in self.identifier_kinds or child.type in _PARAM_PATTERNS
+ ):
+ pattern = child
self._targets(pattern, out, sink)
return tuple(out)
@@ -215,6 +231,11 @@ class TsJsDefUseDialect(BaseDefUseDialect):
self._declare(defs, start, declarator, binds=t != _VAR_DECL)
self._process(declarator.child_by_field_name("value"), defs, uses)
return
+ if t == _CATCH: # a nested scope's handler, walked whole
+ sdu = self.handler_def_use(node)
+ defs.extend(sdu.defs)
+ self._process(node.child_by_field_name("body"), defs, uses)
+ return
if t in self.member_access_kinds:
self.collect_reads(node, uses) # receiver is a read; property is not
return
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice.py
index 2d325c20b..9ca4f5fdd 100644
--- a/packages/core/src/repowise/core/analysis/health/dataflow/slice.py
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice.py
@@ -190,6 +190,8 @@ def find_extractions(
hoisted = _hoisted_bindings(def_lines, use_lines)
decl_lines = _declaration_lines(analysis.def_use)
shared = _closure_state(analysis.def_use, def_lines, use_lines)
+ outer = analysis.def_use.outer
+ carry_defs = _rebinding_defs_dropped(analysis, def_lines, lmap)
scan = _scan_for(lmap, receiver, fn_node)
scope_kinds = scan.scopes
free_writes = _free_write_lines(analysis.def_use) if receiver and receiver.implicit else []
@@ -252,8 +254,8 @@ def find_extractions(
continue
if any(s <= first_def <= e and first_use < s for first_def, first_use in hoisted):
continue
- if _declaration_escapes(
- s, e, returns, decl_lines, def_lines, use_lines, declared_first
+ if _bindings_escape(
+ s, e, returns, decl_lines, def_lines, use_lines, declared_first, outer
):
continue
if pre.nested[j + 1] > pre.nested[i]:
@@ -261,7 +263,7 @@ def find_extractions(
if shared is not None and _closure_state_crosses(shared, span, s, e, block, lmap):
continue
if loop is not None and not _loop_carry_free(
- span, loop, s, e, def_lines, use_lines, lmap
+ span, loop, s, e, carry_defs, use_lines, lmap
):
continue
uses, assigns = _receiver_facts(receiver, pre, i, j, _count_in(free_writes, s, e))
@@ -965,6 +967,40 @@ def _declaration_escapes(
)
+def _bindings_escape(
+ s: int,
+ e: int,
+ returns: tuple[str, ...],
+ decl_lines: dict[str, frozenset[int]],
+ def_lines: dict[str, list[int]],
+ use_lines: dict[str, list[int]],
+ declared_first: dict[str, frozenset[int]],
+ outer: dict[str, frozenset[int]],
+) -> bool:
+ """A binding the span cannot take along: a declaration the code after it
+ still needs (:func:`_declaration_escapes`), or a ``global`` / ``nonlocal``
+ write whose declaration stays behind (:func:`_outer_write_escapes`)."""
+ return _declaration_escapes(
+ s, e, returns, decl_lines, def_lines, use_lines, declared_first
+ ) or _outer_write_escapes(s, e, outer, def_lines)
+
+
+def _outer_write_escapes(
+ s: int, e: int, outer: dict[str, frozenset[int]], def_lines: dict[str, list[int]]
+) -> bool:
+ """True when the span writes a ``global`` / ``nonlocal`` name it cannot
+ take its declaration along for: one declared outside the span, or also
+ written outside it. In a helper without the declaration the write binds a
+ local and the module value never changes."""
+ for var, declared in outer.items():
+ writes = def_lines.get(var, ())
+ if not any(s <= ln <= e for ln in writes):
+ continue
+ if any(not s <= ln <= e for ln in (*declared, *writes)):
+ return True
+ return False
+
+
def _needed_after(
var: str,
e: int,
@@ -1116,6 +1152,36 @@ def _branch_assigns(node: Node, var: str, defs: list[int], lmap: LanguageNodeMap
return _stmts_assign(list(node.named_children), var, defs, lmap)
+def _rebinding_defs_dropped(
+ analysis: FunctionAnalysis, def_lines: dict[str, list[int]], lmap: LanguageNodeMap
+) -> dict[str, list[int]]:
+ """*def_lines* without the names exception handlers bind on their head
+ line (``except E as e:``, ``catch (err)``). Such a binding never reaches
+ the next iteration: Python unbinds it at the handler's end and a catch
+ parameter is scoped to its block, so a read of the name elsewhere in the
+ loop is another handler's own."""
+ heads = [
+ st.node
+ for block in analysis.cfg.blocks
+ for st in block.statements
+ if st.is_head and st.kind in lmap.catch_kinds and st.node is not None
+ ]
+ if not heads:
+ return def_lines
+ bound: set[tuple[str, int]] = set()
+ for head in heads:
+ # Only the clause's own binder: before its body, so a one-line body's
+ # write on the same line still counts.
+ body = next((c for c in head.named_children if c.type in lmap.block_kinds), None)
+ end = body.start_point if body is not None else head.end_point
+ bound |= {
+ (d.var, d.line)
+ for d in analysis.def_use.definitions
+ if head.start_point <= (d.line - 1, d.column) < end
+ }
+ return {var: [ln for ln in lines if (var, ln) not in bound] for var, lines in def_lines.items()}
+
+
def _loop_carry_free(
span: list[Node],
loop: Node,
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/stages.py b/packages/core/src/repowise/core/analysis/health/dataflow/stages.py
index 080656081..9043b9f34 100644
--- a/packages/core/src/repowise/core/analysis/health/dataflow/stages.py
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/stages.py
@@ -49,7 +49,7 @@ from __future__ import annotations
import re
from bisect import bisect_right
from collections import Counter
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from typing import TYPE_CHECKING
from ..complexity.nloc import is_string_stmt
@@ -69,6 +69,7 @@ from .slice import (
_hoisted_bindings,
_identifiers,
_infer_in_out,
+ _outer_write_escapes,
_Prefix,
_reads_observing,
_receiver_facts,
@@ -94,6 +95,9 @@ STAGE_MAX_NLOC = 60
_STAGE_MIN_DECISIONS = 2
# More shared values than a signature carries well (the slicer's own cap).
_CONTEXT_OVER = 5
+# Why a stage that cleared every other check was not offered (``StagePlan.refused``).
+OUTER_WRITE = "outer_write"
+SWITCH_ARM_WRITE = "switch_arm_write"
# A wrapper holding this share of a block's decision points is unwrapped.
_WRAPPER_SHARE = 0.6
# Loop bodies and ``if`` arms looked into below the outermost cut.
@@ -119,13 +123,16 @@ class StagePlan:
object (empty: plain parameters), and per stage the names it reads that
the function imports outside it, which the helper must import itself;
``moved_imports`` are those nothing left in the function reads.
- ``reason`` says why there are no stages; None when there are."""
+ ``reason`` says why there are no stages; None when there are.
+ ``refused`` names why a stage that cleared every other check was not
+ offered (:data:`OUTER_WRITE`, :data:`SWITCH_ARM_WRITE`)."""
stages: tuple[Extraction, ...] = ()
context: tuple[str, ...] = ()
imports: tuple[tuple[str, ...], ...] = ()
moved_imports: frozenset[str] = frozenset()
reason: str | None = None
+ refused: frozenset[str] = frozenset()
@property
def ccn_removed(self) -> int:
@@ -156,7 +163,7 @@ def find_stages(
if body is None:
return StagePlan(reason="no_tree")
blocks = _Blocks(analysis, lmap, receiver, prefixes if prefixes is not None else {})
- facts = _FunctionFacts(analysis)
+ facts = _FunctionFacts(analysis, lmap)
first: StagePlan | None = None
best: StagePlan | None = None
for block, loops in _descent(_unwrap_container(body, lmap.block_kinds), blocks, lmap):
@@ -168,7 +175,7 @@ def find_stages(
# A deeper level wins only by lifting more: ties keep the outer cut.
if plan.stages and (best is None or plan.ccn_removed > best.ccn_removed):
best = plan
- return best or first or StagePlan(reason="no_tree")
+ return replace(best or first or StagePlan(reason="no_tree"), refused=frozenset(facts.refused))
def _plan_block(
@@ -304,7 +311,7 @@ def _arms(st: Node, lmap: LanguageNodeMap) -> list[Node]:
class _FunctionFacts:
"""The per-function facts every block's cut reads, built once."""
- def __init__(self, analysis: FunctionAnalysis) -> None:
+ def __init__(self, analysis: FunctionAnalysis, lmap: LanguageNodeMap) -> None:
self.analysis = analysis
def_use = analysis.def_use
self.def_lines, self.use_lines = _var_lines(def_use)
@@ -312,6 +319,9 @@ class _FunctionFacts:
self.hoisted = _hoisted_bindings(self.def_lines, self.use_lines)
self.decl_lines = _declaration_lines(def_use)
self.shared = _closure_state(def_use, self.def_lines, self.use_lines)
+ self.outer = def_use.outer
+ self.arm_writes = _arm_writes(analysis, lmap)
+ self.refused: set[str] = set()
self.reads = _reads_observing(analysis, self.def_lines)
self.assigned = DefiniteAssignment(analysis)
self.inner_bindings = _inner_bindings(def_use)
@@ -343,6 +353,9 @@ class _Cutter:
self.hoisted = facts.hoisted
self.decl_lines = facts.decl_lines
self.shared = facts.shared
+ self.outer = facts.outer
+ self.arm_writes = facts.arm_writes
+ self.refused = facts.refused
self.reads = facts.reads
self.pre = pre
self._assigns: dict[tuple[int, str], bool] = {}
@@ -426,7 +439,9 @@ class _Cutter:
bound = self.assigned.before(s, e)
if bound is None or not set(params) <= bound:
return None
- if not self._signature_holds(span, s, e, returns) or self._guard_needed_after(i, j, e):
+ if self._guard_needed_after(i, j, e) or not self._signature_holds(
+ span, s, e, params, returns
+ ):
return None
pre = self.pre
uses, assigns = _receiver_facts(self.receiver, pre, i, j, 0)
@@ -484,19 +499,71 @@ class _Cutter:
if names
)
- def _signature_holds(self, span: list[Node], s: int, e: int, returns: tuple[str, ...]) -> bool:
+ def _signature_holds(
+ self,
+ span: list[Node],
+ s: int,
+ e: int,
+ params: tuple[str, ...],
+ returns: tuple[str, ...],
+ ) -> bool:
"""The single-span slicer's own refusals: a hoisted binding, a
declaration the code after the stage still needs, a local a closure
- shares across the stage's edge."""
+ shares across the stage's edge. Then the two a stage names in
+ ``refused``: a ``global`` write that cannot take its declaration
+ along, a ``switch`` / ``match`` arm write the interface depends on."""
if any(s <= first_def <= e and first_use < s for first_def, first_use in self.hoisted):
return False
if _declaration_escapes(
s, e, returns, self.decl_lines, self.def_lines, self.use_lines, self.declared_first
):
return False
- return self.shared is None or not _closure_state_crosses(
+ if self.shared is not None and _closure_state_crosses(
self.shared, span, s, e, self.block, self.lmap
- )
+ ):
+ return False
+ reason = self._interface_refusal(s, e, params, returns)
+ if reason is not None:
+ self.refused.add(reason)
+ return reason is None
+
+ def _interface_refusal(
+ self, s: int, e: int, params: tuple[str, ...], returns: tuple[str, ...]
+ ) -> str | None:
+ if _outer_write_escapes(s, e, self.outer, self.def_lines):
+ return OUTER_WRITE
+ if self._arm_write_crosses(s, e, params, returns):
+ return SWITCH_ARM_WRITE
+ return None
+
+ def _arm_write_crosses(
+ self, s: int, e: int, params: tuple[str, ...], returns: tuple[str, ...]
+ ) -> bool:
+ """Whether a ``switch`` / ``match`` arm's write of a name live across
+ the stage's edge can decide its interface. The CFG keeps the
+ statement whole, so liveness takes the write as certain and hides
+ the value from before it: a read after the arm in the stage misses an
+ input, a stage before the arm misses an output, an input bound only
+ by the arm looks bound. Temporary: remove once the CFG splits arms
+ (#3451)."""
+ taken = set(params)
+ for lo, hi, names in self.arm_writes:
+ for var in names:
+ defs = self.def_lines.get(var, ())
+ uses = self.use_lines.get(var, ())
+ if s <= lo and hi <= e:
+ hazard = var not in taken and any(d < s for d in defs) and any(
+ hi < u <= e for u in uses
+ )
+ elif hi < s:
+ hazard = var in taken and not any(d < lo for d in defs)
+ else:
+ hazard = var not in returns and any(s <= d <= e for d in defs) and any(
+ u > hi for u in uses
+ )
+ if hazard:
+ return True
+ return False
def _with_inouts(
self, i: int, j: int, s: int, params: tuple[str, ...], returns: tuple[str, ...]
@@ -562,6 +629,29 @@ def _rereads(analysis: FunctionAnalysis) -> frozenset[str]:
return frozenset(out)
+def _arm_writes(
+ analysis: FunctionAnalysis, lmap: LanguageNodeMap
+) -> tuple[tuple[int, int, frozenset[str]], ...]:
+ """Per ``switch`` / ``match`` statement the flow graph keeps whole, its
+ lines and the names an arm writes without declaring them."""
+ if not lmap.switch_kinds:
+ return ()
+ found = [
+ (st.start_line, st.end_line)
+ for block in analysis.cfg.blocks
+ for st in block.statements
+ if st.kind in lmap.switch_kinds
+ ]
+ out = []
+ for lo, hi in found:
+ names = frozenset(
+ d.var for d in analysis.def_use.definitions if lo <= d.line <= hi and not d.declares
+ )
+ if names:
+ out.append((lo, hi, names))
+ return tuple(out)
+
+
def _holds_line(node: Node, lines: list[int]) -> bool:
lo, hi = node.start_point[0] + 1, node.end_point[0] + 1
return any(lo <= ln <= hi for ln in lines)
@@ -692,4 +782,11 @@ def _composes(
return tuple(tuple(sorted(names)) for names in imported), moved
-__all__ = ["STAGE_MAX_CCN", "STAGE_MAX_NLOC", "StagePlan", "find_stages"]
+__all__ = [
+ "OUTER_WRITE",
+ "STAGE_MAX_CCN",
+ "STAGE_MAX_NLOC",
+ "SWITCH_ARM_WRITE",
+ "StagePlan",
+ "find_stages",
+]
diff --git a/packages/core/src/repowise/core/analysis/health/refactoring/extract_method.py b/packages/core/src/repowise/core/analysis/health/refactoring/extract_method.py
index ddae51cf4..1a3e94159 100644
--- a/packages/core/src/repowise/core/analysis/health/refactoring/extract_method.py
+++ b/packages/core/src/repowise/core/analysis/health/refactoring/extract_method.py
@@ -209,7 +209,7 @@ class ExtractMethodDetector(RefactoringDetector):
prefixes: dict[int, Any] = {}
candidates = find_extractions(analysis, lmap, receiver, prefixes)
best = _choose(analysis, candidates, markers)
- staged = _staged(analysis, best, lmap, receiver, ctx.language, markers, prefixes)
+ staged = _staged(analysis, best, lmap, receiver, ctx.language, matched, prefixes)
scope = _scope(fn_node, files) if staged is not None or best is not None else None
if staged is not None:
split = self._staged_suggestion(ctx, analysis, staged, matched, names, receiver)
@@ -363,6 +363,20 @@ class ExtractMethodDetector(RefactoringDetector):
REFUSED_DETAIL = "extract_method_refused"
+#: The finding detail naming why a stage that cleared every other check was
+#: not offered (``StagePlan.refused``), sorted.
+STAGE_REFUSED_DETAIL = "extract_method_stage_refused"
+
+
+def _mark_stage_refused(findings: list[Any], reasons: frozenset[str]) -> None:
+ if not reasons:
+ return
+ for f in findings:
+ details = getattr(f, "details", None)
+ if isinstance(details, dict):
+ details[STAGE_REFUSED_DETAIL] = sorted(reasons)
+
+
def _mark_refused(findings: list[Any], reason: str) -> None:
for f in findings:
details = getattr(f, "details", None)
@@ -417,18 +431,22 @@ def _staged(
lmap: LanguageNodeMap,
receiver: Receiver | None,
language: str | None,
- markers: set[str],
+ findings: list[Any],
prefixes: dict[int, Any] | None = None,
) -> StagePlan | None:
"""The staged split, when the function is too complex for one helper and
the split lifts at least twice what the best single span does (several
edits for a point or two more is not the better plan); each stage must
- clear the floor a single span does."""
+ clear the floor a single span does. Why a stage was not offered is kept
+ on *findings* (:data:`STAGE_REFUSED_DETAIL`), so refusals are counted
+ from the index."""
+ markers = {getattr(f, "biomarker_type", "") for f in findings}
# Only languages with a staged renderer, each with the outputs one call binds.
max_returns = render.staged_outputs(language)
if max_returns is None or analysis.ccn < _STAGE_MIN_CCN:
return None
plan = find_stages(analysis, lmap, receiver, max_returns=max_returns, prefixes=prefixes)
+ _mark_stage_refused(findings, plan.refused)
if not plan.stages or (best is not None and plan.ccn_removed < 2 * best.ccn_removed):
return None
if not all(_worth_extracting(analysis, x, markers) for x in plan.stages):
diff --git a/tests/unit/health/test_extract_method_span_interface.py b/tests/unit/health/test_extract_method_span_interface.py
new file mode 100644
index 000000000..05aaa0978
--- /dev/null
+++ b/tests/unit/health/test_extract_method_span_interface.py
@@ -0,0 +1,484 @@
+"""The values a staged or single Extract Method span takes and hands back.
+
+Each case is a shape whose stage interface was wrong on real code (hermes,
+openclaw) when an independent name-use pass re-checked every served stage:
+a parameter the dialect did not see, an ``except`` clause's own reads and
+binding, a ``global`` write, and a ``switch`` / ``match`` arm write the flow
+graph takes as certain.
+"""
+
+from __future__ import annotations
+
+import textwrap
+
+import pytest
+
+from repowise.core.analysis.health.complexity.languages import get_language_map
+from repowise.core.analysis.health.dataflow import find_extractions
+from repowise.core.analysis.health.refactoring.extract_method import (
+ STAGE_REFUSED_DETAIL,
+ ExtractMethodDetector,
+)
+from repowise.core.analysis.health.refactoring.models import RefactoringContext
+from tests.unit.health.test_extract_method_stages import (
+ _functions,
+ _item_section,
+ _loop_fn,
+ _plans,
+ _ts_section,
+)
+
+
+class _Finding:
+ def __init__(self, fn):
+ self.biomarker_type, self.function_name = "complex_method", fn.name
+ self.line_start, self.health_impact, self.details = fn.start_line, 2.0, {}
+
+
+def _refused(src: str, language: str = "python", ext: str = "py") -> dict[str, list[str]]:
+ """Per function, the stage refusal reasons the detector records."""
+ fns = _functions(src, language, ext)
+ findings = [_Finding(f) for f in fns]
+ ctx = RefactoringContext(
+ file_path=f"m.{ext}", language=language, nloc=300, findings=findings, function_analyses=fns
+ )
+ ExtractMethodDetector().detect(ctx)
+ return {f.function_name: f.details.get(STAGE_REFUSED_DETAIL, []) for f in findings}
+
+
+def _spans(src: str, name: str, language: str = "python", ext: str = "py"):
+ fn = next(f for f in _functions(src, language, ext) if f.name == name)
+ return find_extractions(fn, get_language_map(language))
+
+
+def _stage_holding(plan, src: str, text: str) -> dict:
+ lines = textwrap.dedent(src).splitlines()
+ at = next(i for i, ln in enumerate(lines, 1) if ln.strip() == text)
+ return next(s for s in plan.plan["stages"] if s["span"]["start"] <= at <= s["span"]["end"])
+
+
+# -- parameters the signature binds --------------------------------------------
+
+
+def test_a_typed_splat_parameter_is_bound():
+ # hermes ``moa_loop.py::create(self, **api_kwargs: Any)``: the stages read
+ # ``api_kwargs`` and the plan passed nothing.
+ fn = _functions("def f(a, *args: int, b=1, **kw: str):\n return a, args, b, kw\n")[0]
+ assert [p.name for p in fn.def_use.params] == ["a", "args", "b", "kw"]
+ src = _loop_fn(extras={2: "\n note(kw)"}).replace(
+ "opt_7):\n seen", "opt_7, **kw: int):\n seen"
+ )
+ plan = _plans(src)["run"]
+ assert "kw" in _stage_holding(plan, src, "note(kw)")["params"]
+
+
+def test_javascript_default_and_destructured_parameters_are_bound():
+ # openclaw ``runGatewayCpuScenarios(options, params = {})`` and
+ # ``runBarnacleAutoResponse({ github, context })``: stages read them and
+ # the plan passed neither.
+ fn = _functions(
+ "function f(a, b = 1, { c, d = 2 }, [e], ...r) { return a + b + c + d + e + r; }",
+ "javascript",
+ "js",
+ )[0]
+ assert [p.name for p in fn.def_use.params] == ["a", "b", "c", "d", "e", "r"]
+ body = textwrap.indent(textwrap.dedent("".join(_ts_section(n) for n in range(8))), " ")
+ opts = ", ".join(f"opt{n}" for n in range(8))
+ src = (
+ f"function run(items, a = 0, {{ b, seen, {opts} }} = {{}}) {{\n"
+ f" for (const item of items) {{\n{body}\n }}\n return seen;\n}}\n"
+ )
+ stages = _plans(src, "javascript", "js")["run"].plan["stages"]
+ assert len(stages) >= 2
+ for stage in stages:
+ lines = src.splitlines()[stage["span"]["start"] - 1 : stage["span"]["end"]]
+ read = {f"opt{n}" for n in range(8) if any(f"opt{n})" in ln for ln in lines)}
+ assert read <= set(stage["params"])
+ assert "b" in stage["params"] and "seen" in stage["params"]
+
+
+# -- the head of an exception handler --------------------------------------------
+
+
+def test_an_except_clause_reads_its_type_and_binds_its_name():
+ fn = _functions(
+ """
+ def f(x):
+ from m import AuthError
+ try:
+ y = g(x)
+ except AuthError as exc:
+ log(exc)
+ except (ValueError, KeyError):
+ pass
+ """
+ )[0]
+ handlers = [b for b in fn.cfg.blocks if b.kind == "handler"]
+ assert [b.statements[0].start_line for b in handlers] == [6, 8]
+ first, second = (fn.def_use.block(b.id) for b in handlers)
+ assert [(d.var, d.line) for d in first.defs] == [("exc", 6)]
+ assert [(u.name, u.line) for u in first.uses] == [("AuthError", 6), ("log", 7), ("exc", 7)]
+ assert {u.name for u in second.uses} == {"ValueError", "KeyError"}
+
+
+def test_a_closure_handler_binds_its_own_name():
+ # hermes ``run_doctor``: a nested check's ``except Exception as e`` read
+ # as the outer ``e`` once the outer handlers bound one.
+ fn = _functions(
+ """
+ def f(x):
+ try:
+ g(x)
+ except Exception as e:
+ log(e)
+
+ def check():
+ try:
+ h()
+ except OSError as e:
+ log(e)
+ return check
+ """
+ )[0]
+ assert "e" not in {o.name for o in fn.def_use.captured.reads}
+
+
+def test_a_span_in_a_handler_takes_the_bound_exception():
+ # hermes ``gateway/delivery.py::deliver``: the span reading ``str(e)``
+ # inside ``except Exception as e`` did not take ``e``.
+ src = """
+ def deliver(targets, results):
+ for target in targets:
+ try:
+ send(target)
+ except Exception as e:
+ if target.chat_id and target.platform:
+ kind = classify(str(e))
+ if kind:
+ mark(target, kind)
+ results[target.name] = {"error": str(e), "kind": target.kind}
+ return results
+ """
+ fn = _functions(src)[0]
+ spans = find_extractions(fn, get_language_map("python"))
+ assert spans and all(
+ "e" in x.params for x in spans if x.start_line >= 7 and x.end_line <= 11
+ )
+
+
+def test_a_stage_reimports_an_exception_type_the_function_imports():
+ # hermes ``cmd_gateway_enroll``: ``except AuthError`` read a function-local
+ # import and the stage was not told to import it.
+ handler = (
+ "\n try:\n refresh(item)\n"
+ " except AuthError as exc:\n report(exc)"
+ )
+ src = _loop_fn(extras={3: handler}).replace(
+ " seen = []", " from auth import AuthError\n\n seen = []"
+ )
+ plan = _plans(src)["run"]
+ notes = _stage_holding(plan, src, "except AuthError as exc:")["new_symbol"].get("notes", [])
+ assert any(n.startswith("Import AuthError in the helper") for n in notes)
+
+
+# -- names the module or an enclosing function owns ------------------------------
+
+_GLOBAL_FN = """
+def resolve(items, a, b, {opts}):
+ global _CACHE
+ _CACHE = None
+ seen = []
+ last = None
+ for item in items:
+{body}
+ last = item
+ return seen, last
+"""
+
+
+def test_a_global_write_never_leaves_its_declaration():
+ # hermes ``resolve_nous_access_token``: a stage wrote ``_CACHE = ...`` under
+ # the function's ``global`` and the helper would bind a local instead.
+ body = "".join(
+ _item_section(n, extra="\n _CACHE = item" if n == 3 else "") for n in range(8)
+ )
+ src = _GLOBAL_FN.format(
+ opts=", ".join(f"opt_{n}" for n in range(8)),
+ body=textwrap.indent(textwrap.dedent(body), " " * 8),
+ )
+ write = next(i for i, ln in enumerate(src.splitlines(), 1) if ln.strip() == "_CACHE = item")
+ plan = _plans(src)["resolve"]
+ for stage in plan.plan.get("stages") or [plan.plan]:
+ assert not stage["span"]["start"] <= write <= stage["span"]["end"]
+ fn = _functions(src)[0]
+ assert all(
+ not x.start_line <= write <= x.end_line
+ for x in find_extractions(fn, get_language_map("python"))
+ )
+
+
+def test_a_span_holding_the_declaration_and_every_write_is_offered():
+ src = """
+ def warm(keys, flag):
+ n = len(keys)
+ global _CACHE
+ if flag and n:
+ _CACHE = {}
+ for k in keys:
+ _CACHE[k] = n
+ log(n)
+ return n
+ """
+ fn = _functions(src)[0]
+ spans = find_extractions(fn, get_language_map("python"))
+ assert any(x.start_line == 4 and x.end_line == 8 for x in spans)
+
+
+# -- a write in a switch arm ---------------------------------------------------------
+
+
+def _ts_loop(mode_stmt: str) -> str:
+ body = textwrap.indent(textwrap.dedent("".join(_ts_section(n) for n in range(8))), " ")
+ opts = ", ".join(f"opt{n}" for n in range(8))
+ return (
+ f"function run(items, a, b, seen, mode, last, {opts}) {{\n"
+ f" for (const item of items) {{\n{body}\n {mode_stmt}\n"
+ " note(last);\n }\n return seen;\n}\n"
+ )
+
+
+def test_a_stage_never_reads_past_a_switch_arm_write_it_did_not_take():
+ # The flow graph keeps a ``switch`` whole, so its arm's write looks
+ # certain and hides the value from before it: a stage holding the
+ # ``switch`` and the read after it took no ``last``, and the helper read
+ # an unbound name whenever the arm did not run.
+ for stmt in ("if (mode === 1) { last = item; }", "switch (mode) { case 1: last = item; break; }"):
+ src = _ts_loop(stmt)
+ lines = src.splitlines()
+ arm = next(i for i, ln in enumerate(lines, 1) if ln.strip() == stmt)
+ read = arm + 1
+ stages = _plans(src, "typescript", "ts")["run"].plan["stages"]
+ assert len(stages) >= 2
+ for stage in stages:
+ if stage["span"]["start"] <= arm and read <= stage["span"]["end"]:
+ assert "last" in stage["params"], stmt
+
+
+def test_a_global_refusal_is_counted_under_its_reason():
+ body = "".join(
+ _item_section(n, extra="\n _CACHE = item" if n == 3 else "") for n in range(8)
+ )
+ src = _GLOBAL_FN.format(
+ opts=", ".join(f"opt_{n}" for n in range(8)),
+ body=textwrap.indent(textwrap.dedent(body), " " * 8),
+ )
+ assert _refused(src)["resolve"] == ["outer_write"]
+
+
+def test_a_switch_arm_refusal_is_counted_and_a_dead_arm_name_is_not_refused():
+ hazard = _ts_loop("switch (mode) { case 1: last = item; break; }")
+ assert _refused(hazard, "typescript", "ts")["run"] == ["switch_arm_write"]
+ # An arm writing a name nothing reads past the edge decides nothing.
+ quiet = _ts_loop("switch (mode) { case 1: tally = item; break; }").replace(
+ "mode, last,", "mode, last, tally,"
+ )
+ assert _refused(quiet, "typescript", "ts")["run"] == []
+ stages = _plans(quiet, "typescript", "ts")["run"].plan["stages"]
+ arm = next(i for i, ln in enumerate(quiet.splitlines(), 1) if "switch (mode)" in ln)
+ assert any(s["span"]["start"] <= arm <= s["span"]["end"] for s in stages)
+
+
+# -- closures and outer scopes ----------------------------------------------------
+
+_NONLOCAL = """
+def outer(items, flag):
+ count = 0
+
+ def bump():
+ BODY
+
+ for it in items:
+ if it and flag:
+ bump()
+ if it.big:
+ bump()
+ log(it)
+ print(count)
+ return count
+"""
+
+
+def test_a_span_calling_a_closure_that_writes_nonlocal_is_not_offered():
+ # The closure rebinds the function's ``count`` with no declaration in the
+ # function itself: a helper taking ``count`` would print a stale copy.
+ plain = _spans(_NONLOCAL.replace("BODY", "log(1)"), "outer")
+ assert any(x.start_line == 9 for x in plain)
+ assert _spans(_NONLOCAL.replace("BODY", "nonlocal count\n count += 1"), "outer") == []
+
+
+def test_only_the_function_s_own_global_statements_count():
+ # A nested function's ``global`` is its own: the enclosing function's
+ # write of that name stays a local of the enclosing function.
+ src = """
+ def outer(items, flag):
+ _D = 0
+
+ def inner():
+ global _D
+ _D = 1
+
+ for it in items:
+ if it and flag:
+ _D = it
+ if it.big:
+ log(it)
+ _D += 1
+ return _D, inner
+ """
+ fn = _functions(src)[0]
+ assert fn.def_use.outer == {}
+ own = _functions("def f():\n global _C\n _C = 1\n")[0]
+ assert own.def_use.outer == {"_C": frozenset({2})}
+
+
+# -- exception handler heads ---------------------------------------------------------
+
+
+def test_a_read_of_the_exception_name_after_its_handler_takes_it_in():
+ # Python unbinds ``e`` at the handler's end, which is not modelled: the
+ # read after it is an error in the original, and the span is offered with
+ # ``e`` as an input (documented over-supply, never a lost value).
+ src = """
+ def f(items, flag):
+ total = len(items)
+ try:
+ run(items)
+ except ValueError as e:
+ log(e)
+ print(total)
+ if flag and items:
+ note(e)
+ if items[0] or flag:
+ warn(e)
+ count(items)
+ done(items)
+ return items
+ """
+ spans = [x for x in _spans(src, "f") if x.start_line >= 8]
+ assert spans and all("e" in x.params for x in spans)
+
+
+_TS_CATCH = """function f(items, flag) {
+ for (const it of items) {
+ try {
+ run(it);
+ } catch ({ message }) {
+ if (flag && it.big) {
+ log(message);
+ if (it.small || flag) {
+ warn(message, it);
+ }
+ }
+ report(message);
+ }
+ }
+ return items;
+}
+"""
+
+
+def test_typescript_catch_binds_its_parameter_pattern():
+ fn = _functions(_TS_CATCH, "typescript", "ts")[0]
+ # The handler entry turns ``branch`` when an ``if`` opens it: find it by its head.
+ handler = next(b for b in fn.cfg.blocks if b.statements and b.statements[0].kind == "catch_clause")
+ head = fn.def_use.block(handler.id)
+ assert [(d.var, d.declares) for d in head.defs] == [("message", True)]
+ spans = _spans(_TS_CATCH, "f", "typescript", "ts")
+ assert spans and all("message" in x.params for x in spans)
+
+
+def test_a_closure_catch_parameter_is_its_own():
+ fn = _functions(
+ """function f(xs) {
+ try { run(xs); } catch (err) { log(err); }
+ const g = () => {
+ try {
+ run();
+ } catch (err) {
+ log(err);
+ }
+ };
+ return g;
+}
+""",
+ "typescript",
+ "ts",
+ )[0]
+ captured = fn.def_use.captured
+ assert "err" not in {o.name for o in (*captured.reads, *captured.shared, *captured.writes)}
+
+
+def test_except_star_and_nested_handlers_get_heads():
+ fn = _functions(
+ """
+ def f(x):
+ try:
+ g(x)
+ except* ValueError as group:
+ try:
+ h(group)
+ except KeyError as inner:
+ log(inner, group)
+ """
+ )[0]
+ heads = {
+ (b.statements[0].start_line, b.statements[0].is_head)
+ for b in fn.cfg.blocks
+ if b.kind == "handler"
+ }
+ assert heads == {(5, True), (8, True)}
+ defs = {(d.var, d.line) for d in fn.def_use.definitions}
+ assert {("group", 5), ("inner", 8)} <= defs
+
+
+ ("language", "ext", "src"),
+ [
+ ("java", "java", "class A { int f(int x) { try { g(x); } catch (Exception e) { h(e); } return x; } }"),
+ ("csharp", "cs", "class A { int F(int x) { try { G(x); } catch (Exception e) { H(e); } return x; } }"),
+ ],
+)
+def test_other_languages_read_nothing_new_at_a_handler_head(language, ext, src):
+ fn = _functions(src, language, ext)[0]
+ handler = next(b for b in fn.cfg.blocks if b.kind == "handler")
+ head = handler.statements[0]
+ assert head.is_head
+ bdu = fn.def_use.block(handler.id)
+ assert [d for d in bdu.defs if d.line == head.start_line] == []
+ assert [u for u in bdu.uses if u.line == head.start_line and u.name == "Exception"] == []
+
+
+def test_a_handler_rebinding_a_name_in_a_loop_carries_nothing():
+ # hermes ``_install_memory_provider_external_dependencies``: each handler
+ # binds its own ``exc``; the span holding the second one is not loop-carried.
+ src = """
+ def install(deps, results):
+ for dep in deps:
+ try:
+ setup(dep)
+ except Exception as exc:
+ results.append(str(exc))
+ if dep.check and dep.name:
+ try:
+ verify(dep)
+ if dep.slow or dep.big:
+ wait(dep)
+ except Exception as exc:
+ results.append(str(exc))
+ log(dep)
+ record(dep, results)
+ notify(dep)
+ flush(results)
+ return results
+ """
+ assert any(x.start_line == 8 for x in _spans(src, "install"))