Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(health): split the Extract Method slicer and the Python perf dialect into focused modules (#4025)
Raghav Chamadiyaa10d3d62734e
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice.pyindex c6eb59363..4102e737b 100644--- a/packages/core/src/repowise/core/analysis/health/dataflow/slice.py+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice.py@@ -1,9 +1,8 @@"""Extract Method slicing: find safe, single-exit spans with IN/OUT inference.-The first user-facing consumer of the dataflow layer. Given a function's-analysis (CFG + def/use from D2), it finds contiguous statement spans that can-be lifted into a helper method without changing behaviour, and infers the-helper's signature:+Given a function's analysis (CFG + def/use), it finds contiguous statement+spans that can be lifted into a helper method without changing behaviour, and+infers the helper's signature:- **IN (parameters)** -- variables the span *reads* whose value is producedbefore the span (defined-before, used-inside).@@ -19,13 +18,13 @@ matter without being nearly the whole body, and has at most one return and asmall parameter list. Everything else is suppressed -- ten great extractions,not two hundred maybes.-Line-based liveness over D2's def/use occurrences realises the IN inference,-reaching definitions over the CFG the OUT inference, and the CFG/jump scan the-single-exit predicate. The jump and nested-scope node kinds come from the-language's ``LanguageNodeMap`` (the same source the CFG builder uses), so every-full-tier language whose map populates them is served by this one slicer.+Node kinds come from the language's ``LanguageNodeMap``, so one slicer serves+every full-tier language. The helpers live in the ``slice_*`` modules and are+re-exported here; patch a name where it is looked up."""+# ruff: noqa: F401+from __future__ import annotationsfrom bisect import bisect_left, bisect_right@@ -39,16 +38,88 @@ from ..complexity.nloc import _code_line_numbersfrom .dialects.base import SUPER, mentions_receiverfrom .raises import exception_flow_refusalfrom .reaching import definitions_at, observed_definitions+from .slice_liveness import (+ _bindings_escape,+ _declaration_escapes,+ _declaration_lines,+ _declared_before_read,+ _hoisted_bindings,+ _import_reads,+ _infer_in_out,+ _needed_after,+ _outer_write_escapes,+ _reads_observing,+ _statements_by_line,+ _valueless_lines,+ _var_lines,+)+from .slice_nodes import (+ _BINDER_FIELDS,+ _awaits,+ _binder_names,+ _descend,+ _exit_macros,+ _function_lines,+ _holds_a_named_nested_function,+ _identifiers,+ _loop_header_names,+ _receiver_root,+ _unwrap_container,+ all_blocks,+)+from .slice_receiver import (+ _FUNCTION_KINDS,+ _INDEX_KINDS,+ _UNPACK_KINDS,+ _UNREAD_TARGET,+ FunctionFacts,+ _collect_assigned,+ _ends_in_exit,+ _free_write_lines,+ _function_kinds,+ _is_receiver,+ _receiver_facts,+ _receiver_field,+ _receiver_kinds,+ _receiver_ref,+ _ReceiverSink,+ _whole_receiver_facts,+ body_tally,+ function_facts,+)+from .slice_state import (+ _closure_state,+ _closure_state_crosses,+ _count_in,+ _declared_above,+ _declares_in,+ _loop_carry_free,+ _rebinding_defs_dropped,+ _SharedState,+ _span_binds,+ _span_only_rebinds,+)+from .slice_writes import (+ _branch_assigns,+ _conditional_assigns,+ _is_conditional,+ _outs_definitely_assigned,+ _stmt_assigns,+ _stmts_assign,+ _unconditional_write,+ _writes_var,+)if TYPE_CHECKING:+ from collections.abc import Iterator+from tree_sitter import Nodefrom ..complexity.languages import LanguageNodeMapfrom .analyze import FunctionAnalysis- from .cfg import CFG- from .defuse import FunctionDefUsefrom .dialects.base import Receiver+# Gates (precision-first; tuned to suppress trivial or unwieldy extractions)._MIN_STMTS = 2 # at least two statements# The extracted helper is substantial: code lines, the walker's NLOC rule, so a@@ -56,15 +127,12 @@ _MIN_STMTS = 2 # at least two statements_MIN_SLICE_NLOC = 5_MIN_CCN_REMOVED = 1 # extraction must remove a real decision point_MAX_PARAMS = 5 # too many ins => the span is not cohesive-_MAX_RETURNS = 1 # a single clean return (v1); multi-output is future work+_MAX_RETURNS = 1 # a single clean return# Backstop against a pathological function producing too many sub-ranges._MAX_CANDIDATES = 4000# A span holding this share of the body's code lines or more is the whole-# function under another name, not a split: what stays behind is a guard, a-# wrapper (``try`` / ``with``) or the final ``return``. Measured on 301 plans-# from seven repos (Python, TS, Go, Java, Rust): 52 sat at or above 0.75, and-# nearly all of those at 0.8 or more left only such a shell. The span offered-# instead may be the same block minus a statement, just under the cut.+# function under another name: what stays behind is only a guard, a wrapper or+# the final ``return``._MAX_BODY_SHARE = 0.75@@ -157,149 +225,229 @@ def find_extractions("""Return safe Extract Method candidates for *analysis*, best first.Best is most complexity removed, then largest span, then fewest parameters,- then earliest -- a deterministic order. Empty when the function has no AST- node retained or no span clears the extractability gates. *receiver* is- the function's instance (``DefUseDialect.receiver``), None when unknown.+ then earliest. *receiver* is the function's instance, None when unknown.*prefixes*, when given, receives each walked block's :class:`_Prefix` by- block id, so a staged plan reads them instead of walking again.- *refused*, when given, receives why a span that cleared every other gate- was dropped because lifting it would change exception flow- (:func:`raises.exception_flow_refusal`).+ block id; *refused* receives why a span that cleared every other gate would+ change exception flow (:func:`raises.exception_flow_refusal`)."""fn_node = analysis.fn_node- if fn_node is None:- return []- # A subtree tree-sitter could not parse is not a function whose statements- # we understand, so no span in it is safe to lift. This fires on macro-heavy- # C/C++ headers, where an unterminated function-like macro- # (``ABSL_NAMESPACE_BEGIN``) makes the parser emit one bogus- # ``function_definition`` spanning a whole class or namespace — proposing to- # extract "statements" from that is a wrong suggestion, not a weak one.- # Language-agnostic and strictly subtractive: a clean parse is unaffected.- if fn_node.has_error:+ # A subtree tree-sitter could not parse (a macro-heavy C/C++ header emitting+ # one bogus function over a whole namespace) has no statements we understand.+ if fn_node is None or fn_node.has_error:return []body = fn_node.child_by_field_name("body")if body is None:return []- # The statement container of the body (Go nests it in a ``statement_list``- # inside the ``block``); spans covering it whole are not extractions.- body_container = _unwrap_container(body, lmap.block_kinds)- lines = _function_lines(fn_node)- max_slice_nloc = _MAX_BODY_SHARE * _stmts_nloc(body_container.named_children, lines)-- def_lines, use_lines = _var_lines(analysis.def_use)- import_reads = _import_reads(analysis)- declared_first = _declared_before_read(analysis.def_use)- valueless = _valueless_lines(analysis.def_use)- # Built on the first span that clears the cheap gates.- reads: dict[str, list[tuple[int, frozenset[int]]]] | None = None- hoisted = _hoisted_bindings(def_lines, use_lines)- decl_lines = _declaration_lines(analysis.def_use)- shared = _closure_state(analysis.def_use, def_lines, use_lines)- outer = analysis.def_use.outer- carry_defs = _rebinding_defs_dropped(analysis, def_lines, lmap)- scan = _scan_for(lmap, receiver, fn_node)- scope_kinds = scan.scopes- free_writes = _free_write_lines(analysis.def_use) if receiver and receiver.implicit else []- # Expression-oriented grammars (nonempty ``statement_wrapper_kinds``): a- # block's last child that is not a statement is its tail expression -- the- # block's implicit value. A span ending on one would silently drop that- # value when lifted into a helper, so such spans are refused outright.- tail_stmt_kinds = (- lmap.statement_wrapper_kinds | lmap.local_decl_kinds- if lmap.statement_wrapper_kinds- else None- )-+ slicer = _FunctionSlicer(analysis, lmap, receiver, fn_node, body)out: list[Extraction] = []- evaluated = 0- for block, loop in all_blocks(fn_node, lmap.block_kinds, scope_kinds, lmap.loop_kinds):+ for block, loop in all_blocks(fn_node, lmap.block_kinds, slicer.scan.scopes, lmap.loop_kinds):+ if not slicer.block_extractions(block, loop, prefixes, refused, out):+ break+ return _sorted(out)+++class _Block(NamedTuple):+ """One statement container: its statements and per-block facts."""++ node: Node+ loop: Node | None+ stmts: list[Node]+ pre: _Prefix+ resources: list[int]+++class _Span(NamedTuple):+ """Statements ``i..j`` of a block, spanning lines ``start..end``."""++ i: int+ j: int+ nodes: list[Node]+ start: int+ end: int+++class _FunctionSlicer:+ """The per-function facts every candidate span is checked against, built once."""++ def __init__(+ self,+ analysis: FunctionAnalysis,+ lmap: LanguageNodeMap,+ receiver: Receiver | None,+ fn_node: Node,+ body: Node,+ ) -> None:+ self.analysis = analysis+ self.lmap = lmap+ self.receiver = receiver+ self.fn_node = fn_node+ # Go nests the statements in a ``statement_list``; spans covering it whole are not extractions.+ self.body_container = _unwrap_container(body, lmap.block_kinds)+ self.lines = _function_lines(fn_node)+ self.max_slice_nloc = _MAX_BODY_SHARE * _stmts_nloc(+ self.body_container.named_children, self.lines+ )+ def_use = analysis.def_use+ self.def_lines, self.use_lines = _var_lines(def_use)+ self.import_reads = _import_reads(analysis)+ self.declared_first = _declared_before_read(def_use)+ self.valueless = _valueless_lines(def_use)+ # Built on the first span that clears the cheap gates.+ self.reads: dict[str, list[tuple[int, frozenset[int]]]] | None = None+ self.hoisted = _hoisted_bindings(self.def_lines, self.use_lines)+ self.decl_lines = _declaration_lines(def_use)+ self.shared = _closure_state(def_use, self.def_lines, self.use_lines)+ self.outer = def_use.outer+ self.carry_defs = _rebinding_defs_dropped(analysis, self.def_lines, lmap)+ self.scan = _scan_for(lmap, receiver, fn_node)+ self.free_writes = _free_write_lines(def_use) if receiver and receiver.implicit else []+ # Expression-oriented grammars: a block's last non-statement child is its+ # value, and a span ending on it would drop that value.+ self.tail_stmt_kinds = (+ lmap.statement_wrapper_kinds | lmap.local_decl_kinds+ if lmap.statement_wrapper_kinds+ else None+ )+ self.evaluated = 0+ self.exhausted = False++ def block_extractions(+ self,+ block: Node,+ loop: Node | None,+ prefixes: dict[int, _Prefix] | None,+ refused: set[str] | None,+ out: list[Extraction],+ ) -> bool:+ """Append *block*'s candidates to *out*; False once the candidate budget is spent."""+ lmap = self.lmapstmts = block.named_children- # A match/switch body holds the arms, not statements: a span of them is- # neither a function body nor a valid call site. Each arm's own body is- # still a block of its own.+ # A match/switch body holds arms, not statements; each arm's body is a block of its own.if any(stmt.type in lmap.case_kinds for stmt in stmts):- continue+ return Truen = len(stmts)resources = _scoped_resource_indexes(stmts, lmap)- is_body = block.id == body_container.id- keeps_tail = tail_stmt_kinds is not None and _tail_is_block_value(- stmts, tail_stmt_kinds, fn_node, lmap+ keeps_tail = self.tail_stmt_kinds is not None and _tail_is_block_value(+ stmts, self.tail_stmt_kinds, self.fn_node, lmap)- if n >= _MIN_STMTS:- pre = _block_prefix(stmts, scan, lines, lmap)- if prefixes is not None:- prefixes[block.id] = pre+ spans = self._span_bounds(n, block.id == self.body_container.id, keeps_tail)+ if n < _MIN_STMTS:+ # No span is long enough, but each still counts against the budget.+ for _ in spans:+ pass+ return not self.exhausted+ pre = _block_prefix(stmts, self.scan, self.lines, lmap)+ if prefixes is not None:+ prefixes[block.id] = pre+ blk = _Block(block, loop, stmts, pre, resources)+ for i, j in spans:+ decisions = pre.decisions[j + 1] - pre.decisions[i]+ if pre.jumps[j + 1] > pre.jumps[i] or decisions < _MIN_CCN_REMOVED:+ continue+ slice_nloc = pre.code[j + 1] - pre.code[i]+ if not _MIN_SLICE_NLOC <= slice_nloc < self.max_slice_nloc:+ continue+ found = self._extraction(blk, i, j, decisions, slice_nloc, refused)+ if found is not None:+ out.append(found)+ return not self.exhausted++ def _span_bounds(self, n: int, is_body: bool, keeps_tail: bool) -> Iterator[tuple[int, int]]:+ """Ranges ``(i, j)`` long enough to lift, never the whole body or a block's+ value tail. Every range counts against the candidate budget."""for i in range(n):for j in range(i, n):- evaluated += 1- if evaluated > _MAX_CANDIDATES:- return _sorted(out)+ self.evaluated += 1+ if self.evaluated > _MAX_CANDIDATES:+ self.exhausted = True+ returnlength = j - i + 1- if length < _MIN_STMTS:- continue- # Never extract the whole function body (that is not a split).- if is_body and length == n:- continue- if keeps_tail and j == n - 1:- continue- decisions = pre.decisions[j + 1] - pre.decisions[i]- has_jump = pre.jumps[j + 1] > pre.jumps[i]- if has_jump or decisions < _MIN_CCN_REMOVED:- continue- slice_nloc = pre.code[j + 1] - pre.code[i]- if not _MIN_SLICE_NLOC <= slice_nloc < max_slice_nloc:- continue- span = stmts[i : j + 1]- s = span[0].start_point[0] + 1- e = span[-1].end_point[0] + 1- if any(import_line < s <= use_line <= e for import_line, use_line in import_reads):- continue- if reads is None:- reads = _reads_observing(analysis, def_lines)- params, returns = _infer_in_out(- def_lines, use_lines, s, e, declared_first, reads=reads, valueless=valueless- )- if len(params) > _MAX_PARAMS or len(returns) > _MAX_RETURNS:- continue- if not _outs_definitely_assigned(span, returns, def_lines, lmap):- continue- if any(s <= first_def <= e and first_use < s for first_def, first_use in hoisted):- continue- if _bindings_escape(- s, e, returns, decl_lines, def_lines, use_lines, declared_first, outer- ):- continue- if _disposes_early(resources, i, j, pre.code):- continue- if pre.nested[j + 1] > pre.nested[i]:- continue- if shared is not None and _closure_state_crosses(shared, span, s, e, block, lmap):+ if length < _MIN_STMTS or (is_body and length == n) or (keeps_tail and j == n - 1):continue- if loop is not None and not _loop_carry_free(- span, loop, s, e, carry_defs, use_lines, lmap- ):- continue- reason = exception_flow_refusal(analysis.cfg, s, e, reads, def_lines)- if reason is not None and refused is not None:- refused.add(reason)- if reason is not None:- continue- uses, assigns = _receiver_facts(receiver, pre, i, j, _count_in(free_writes, s, e))- out.append(- Extraction(- start_line=s,- end_line=e,- params=params,- returns=returns,- slice_nloc=slice_nloc,- ccn_removed=decisions,- needs_async=pre.awaits[j + 1] > pre.awaits[i],- uses_receiver=uses,- receiver_assigns=assigns,- )- )- return _sorted(out)+ yield i, j++ def _extraction(+ self,+ blk: _Block,+ i: int,+ j: int,+ decisions: int,+ slice_nloc: int,+ refused: set[str] | None,+ ) -> Extraction | None:+ nodes = blk.stmts[i : j + 1]+ span = _Span(i, j, nodes, nodes[0].start_point[0] + 1, nodes[-1].end_point[0] + 1)+ interface = self._interface(span)+ if interface is None or not self._self_contained(blk, span, interface[1]):+ return None+ s, e = span.start, span.end+ reason = exception_flow_refusal(self.analysis.cfg, s, e, self.reads, self.def_lines)+ if reason is not None:+ if refused is not None:+ refused.add(reason)+ return None+ params, returns = interface+ pre = blk.pre+ free_writes = _count_in(self.free_writes, s, e)+ uses, assigns = _receiver_facts(self.receiver, pre, i, j, free_writes)+ return Extraction(+ start_line=s,+ end_line=e,+ params=params,+ returns=returns,+ slice_nloc=slice_nloc,+ ccn_removed=decisions,+ needs_async=pre.awaits[j + 1] > pre.awaits[i],+ uses_receiver=uses,+ receiver_assigns=assigns,+ )++ def _interface(self, span: _Span) -> tuple[tuple[str, ...], tuple[str, ...]] | None:+ """The span's ``(params, returns)``, or None when they are too many or an+ OUT is not written on every path."""+ s, e = span.start, span.end+ if any(import_line < s <= use_line <= e for import_line, use_line in self.import_reads):+ return None+ if self.reads is None:+ self.reads = _reads_observing(self.analysis, self.def_lines)+ params, returns = _infer_in_out(+ self.def_lines,+ self.use_lines,+ s,+ e,+ self.declared_first,+ reads=self.reads,+ valueless=self.valueless,+ )+ if len(params) > _MAX_PARAMS or len(returns) > _MAX_RETURNS:+ return None+ if not _outs_definitely_assigned(span.nodes, returns, self.def_lines, self.lmap):+ return None+ return params, returns++ def _self_contained(self, blk: _Block, span: _Span, returns: tuple[str, ...]) -> bool:+ """No binding, resource, nested function, closure state or loop-carried+ value ties the span to code it does not hold."""+ s, e = span.start, span.end+ if any(s <= first_def <= e and first_use < s for first_def, first_use in self.hoisted):+ return False+ if _bindings_escape(+ s, e, returns, self.decl_lines, self.def_lines, self.use_lines,+ self.declared_first, self.outer,+ ):+ return False+ pre = blk.pre+ if _disposes_early(blk.resources, span.i, span.j, pre.code):+ return False+ if pre.nested[span.j + 1] > pre.nested[span.i]:+ return False+ if self.shared is not None and _closure_state_crosses(+ self.shared, span.nodes, s, e, blk.node, self.lmap+ ):+ return False+ return blk.loop is None or _loop_carry_free(+ span.nodes, blk.loop, s, e, self.carry_defs, self.use_lines, self.lmap+ )def _scoped_resource_indexes(stmts: list[Node], lmap: LanguageNodeMap) -> list[int]:@@ -323,119 +471,6 @@ def _disposes_early(resources: list[int], i: int, j: int, code: list[int]) -> boreturn code[-1] > code[j + 1] and any(i <= k <= j for k in resources)-@dataclass(frozen=True)-class FunctionFacts:- """What a whole function does, counted on the complexity walk's visits.-- ``awaits``: it suspends outside any nested scope. ``is_generator``: it- yields. ``uses_receiver`` / ``receiver_assigns`` as on :class:`Extraction`,- for the whole body; ``receiver_assigns`` is None where an implicit receiver- (Java, C++) can write fields by bare name, which only def/use can tell.- ``early_exits``: returns and raises other than a final one.- """-- awaits: bool- is_generator: bool- uses_receiver: bool | None- receiver_assigns: tuple[str, ...] | None- early_exits: int---class _ReceiverSink:- """``BodyTally.on_receiver``: whether the body names its receiver, and- which of its fields it assigns."""-- __slots__ = ("assigns", "receiver", "uses")-- def __init__(self, receiver: Receiver) -> None:- self.receiver = receiver- self.uses = False- self.assigns: set[str] = set()-- def __call__(self, node: Node) -> None:- self.uses = _receiver_ref(node, self.receiver, self.assigns) or self.uses---def body_tally(fn_node: Node, lmap: LanguageNodeMap, receiver: Receiver | None) -> BodyTally:- """An empty tally for *fn_node*, for the complexity walk to fill. The- receiver is watched only when the function's text names it."""- watched = bool(- receiver is not None and receiver.names and mentions_receiver(fn_node, receiver.names)- )- sink = _ReceiverSink(receiver) if watched and receiver is not None else None- return BodyTally(_function_kinds(lmap, sink), sink)---# Built once per language map and receiver shape: the walk asks per function.-_FUNCTION_KINDS: dict[tuple[int, frozenset[str]], FactKinds] = {}---def _function_kinds(lmap: LanguageNodeMap, sink: _ReceiverSink | None) -> FactKinds:- receiver = _receiver_kinds(sink.receiver) if sink else frozenset()- key = (id(lmap), receiver)- kinds = _FUNCTION_KINDS.get(key)- if kinds is None:- kinds = _FUNCTION_KINDS[key] = FactKinds(- exits=lmap.return_kinds | lmap.raise_kinds,- exit_macros=_exit_macros(lmap),- yields=lmap.yield_kinds,- awaits=lmap.await_kinds,- await_scopes=lmap.await_scope_kinds,- lambdas=lmap.lambda_kinds,- receiver=receiver,- )- return kinds---def _receiver_kinds(receiver: Receiver) -> frozenset[str]:- """What :func:`_receiver_ref` can answer for: a write, or a receiver leaf."""- return receiver.write_kinds | receiver.names | {SUPER, "identifier"}---def function_facts(- fn_node: Node, lmap: LanguageNodeMap, receiver: Receiver | None, tally: BodyTally-) -> FunctionFacts:- """:class:`FunctionFacts` from a filled :func:`body_tally`."""- body = fn_node.child_by_field_name("body") or fn_node- sink = tally.on_receiver if isinstance(tally.on_receiver, _ReceiverSink) else None- uses, assigns = _whole_receiver_facts(- receiver, sink.uses if sink else False, frozenset(sink.assigns) if sink else frozenset()- )- return FunctionFacts(- awaits=tally.awaits,- is_generator=tally.yields > 0,- uses_receiver=uses,- receiver_assigns=assigns,- early_exits=max(0, tally.exits - _ends_in_exit(body, lmap, tally.kinds)),- )---def _ends_in_exit(body: Node, lmap: LanguageNodeMap, kinds: FactKinds) -> bool:- """Whether a block body's last statement leaves the function, so that exit- is not an early one. The statement is the exit (Python ``return x``, TS- ``throw e;``) or wraps it as its first named child (Rust ``return x;`` and- ``bail!(..);`` are expression statements). An expression body has no- statements, so its exits are early by definition and none are subtracted."""- if body.type not in lmap.block_kinds or not body.named_children:- return False- tail = body.named_children[-1]- if is_exit(tail, kinds):- return True- return bool(tail.named_children) and is_exit(tail.named_children[0], kinds)---def _whole_receiver_facts(- receiver: Receiver | None, uses: bool, assigned: frozenset[str]-) -> tuple[bool | None, tuple[str, ...] | None]:- if receiver is None:- return None, None- if not receiver.names:- return False, ()- if receiver.implicit:- return (True if uses else None), None- return uses, None if _UNREAD_TARGET in assigned else tuple(sorted(assigned))--def _scan_for(lmap: LanguageNodeMap, receiver: Receiver | None, fn_node: Node) -> _Scan:"""The walk's kinds for *fn_node*. The receiver is looked for only whenits name (or ``super``) appears in the function's text at all: most@@ -486,34 +521,6 @@ def _block_prefix(return pre-def _free_write_lines(def_use: FunctionDefUse) -> list[int]:- """Sorted lines writing a name the function neither declares nor takes- as a parameter: in Java or C++ that is a field (or a C++ global) written- without ``this``, so the span's receiver writes are not all known."""- params = {p.name for p in def_use.params}- declared = {d.var for d in def_use.definitions if d.declares or d.declared_at is not None}- return sorted(- d.line for d in def_use.definitions if d.var not in params and d.var not in declared- )---def _receiver_facts(- receiver: Receiver | None, pre: _Prefix, i: int, j: int, free_writes: int-) -> tuple[bool | None, tuple[str, ...] | None]:- """``uses_receiver`` / ``receiver_assigns`` for the span over statements- ``i..j`` (see :class:`Extraction`)."""- if receiver is None:- return None, None- if not receiver.names:- return False, ()- named = pre.receiver[j + 1] > pre.receiver[i]- found = frozenset().union(*pre.assigns[i : j + 1])- fields = None if _UNREAD_TARGET in found else tuple(sorted(found))- if receiver.implicit:- return (True if named else None), (None if free_writes else fields)- return named, fields--def _tail_is_block_value(stmts: list[Node], tail_stmt_kinds: frozenset[str], fn_node: Node, lmap: LanguageNodeMap) -> bool:@@ -566,16 +573,6 @@ def _value_hop(node: Node, parent: Node, lmap: LanguageNodeMap) -> bool | None:return True # a let initializer, an argument, an operand-def _function_lines(fn_node: Node) -> list[str]:- """Source rows indexed by absolute row, rebuilt from the function's own text.-- Lets the walker's NLOC rule (``_code_line_numbers``) run here without the- file's bytes; rows above the function are never read, so they stay empty.- """- text = (fn_node.text or b"").decode("utf-8", errors="replace")- return [""] * fn_node.start_point[0] + text.splitlines()--def _stmts_nloc(stmts: list[Node], lines: list[str]) -> int:"""Code lines of *stmts* by the walker's NLOC rule, summed per statement."""return sum(len(_code_line_numbers(st, lines, drop_docstrings=True)) for st in stmts)@@ -588,862 +585,6 @@ def _sorted(candidates: list[Extraction]) -> list[Extraction]:)-def _var_lines(def_use: FunctionDefUse) -> tuple[dict[str, list[int]], dict[str, list[int]]]:- """Per-variable sorted def lines and use lines from D2's facts.-- Parameter definitions are included (seeded at the signature line), so a- parameter naturally counts as "defined before" any body span. Reads inside- nested closures (``def_use.captured.reads``) count as uses at their own line.- An import is not a def here: a helper re-imports a name rather than take- or return it, and :func:`_declaration_escapes` refuses a span holding an- import that later code still reads.- """- def_lines: dict[str, list[int]] = defaultdict(list)- use_lines: dict[str, list[int]] = defaultdict(list)- imported: set[str] = set()- for d in def_use.definitions:- if d.imports:- imported.add(d.var)- else:- def_lines[d.var].append(d.line)- for bdu in def_use.blocks.values():- for u in bdu.uses:- # A may-def's paired use is bookkeeping, not a read: counted, a- # binder declared inside a ``match`` arm became its own parameter.- if not u.echo:- use_lines[u.name].append(u.line)- # A closure's read counts where the closure is written: lifting the code- # around it moves the read with it. Only names this function binds matter.- for u in def_use.captured.reads:- if u.name in def_lines or u.name in imported:- use_lines[u.name].append(u.line)- for lines in def_lines.values():- lines.sort()- for lines in use_lines.values():- lines.sort()- return def_lines, use_lines---def _import_reads(analysis: FunctionAnalysis) -> tuple[tuple[int, int], ...]:- """Function-local import lines paired with the reads they reach."""- imports = {- (definition.var, definition.line)- for definition in analysis.reaching.definitions- if definition.imports- }- if not imports:- return ()- import_lines: dict[str, list[int]] = defaultdict(list)- for name, line in sorted(imports):- import_lines[name].append(line)- return tuple(- (definition_line, use_line)- for name, reads in _reads_observing(analysis, import_lines).items()- for use_line, seen in reads- for definition_line in seen- if (name, definition_line) in imports- )---def _declared_before_read(def_use: FunctionDefUse) -> dict[str, frozenset[int]]:- """Per variable, the lines where a declaration of it comes before its first- read on the same line.-- Lines alone cannot order a write and a read that share one. A C-style- ``for (int i = 0; i < n; i++)`` declares ``i`` and then reads it, while- ``total = total + a[i]`` reads ``total`` and then writes it. Only a- declaration is ordered here, by where its declarator ends: a read past that- point sees the new name, and a read inside the declaration's own- initializer (Go's ``x := x + 1`` in an inner scope) still sees the outer- one. A plain assignment keeps the line rule.- """- first_read: dict[tuple[str, int], int] = {}- for bdu in def_use.blocks.values():- for u in bdu.uses:- if u.echo:- continue- key = (u.name, u.line)- first_read[key] = min(u.column, first_read.get(key, u.column))- declared: dict[str, set[int]] = defaultdict(set)- for d in def_use.definitions:- # No read on the line sorts before any declarator, so it never matches.- if d.declared_at is not None and d.declared_at <= first_read.get((d.var, d.line), -1):- declared[d.var].add(d.line)- return {var: frozenset(lines) for var, lines in declared.items()}---def _valueless_lines(def_use: FunctionDefUse) -> dict[str, frozenset[int]]:- """Per variable, the lines whose only definitions of it are declarations- that give it no value (``Definition.valueless``): Java's ``List<T> xs;``,- TypeScript's ``let x: T;``. A line that also assigns the name- (``int x; x = 1;``) is not one: what a read sees from it is the value."""- bare: dict[str, set[int]] = defaultdict(set)- valued: dict[str, set[int]] = defaultdict(set)- for d in def_use.definitions:- (bare if d.valueless else valued)[d.var].add(d.line)- return {var: frozenset(at - valued[var]) for var, at in bare.items()}---def _reads_observing(- analysis: FunctionAnalysis, def_lines: dict[str, list[int]]-) -> dict[str, list[tuple[int, frozenset[int]]]]:- """Per variable, each read's line with the def lines it may observe.-- Reads take reaching definitions over the CFG, so a later write on a path- that need not run (an ``else`` arm, a ``try`` body that may raise first)- does not hide the span's value from the read. A closure's read observes- what reaches the statement that creates the closure, or, where no recorded- statement covers it, the nearest def above it: a read is never dropped,- since a dropped read can only drop a return. Only names in *def_lines*- count for closures, as in :func:`_var_lines`.- """- def_use, reaching = analysis.def_use, analysis.reaching- defs = reaching.definitions- reads: dict[str, list[tuple[int, frozenset[int]]]] = defaultdict(list)- for use, seen in observed_definitions(def_use, reaching):- if not use.echo:- reads[use.name].append((use.line, frozenset(defs[i].line for i in seen)))- statements: dict[int, tuple[int, int]] | None = None- for u in def_use.captured.reads:- dl = def_lines.get(u.name)- if not dl:- continue- if statements is None:- statements = _statements_by_line(analysis.cfg)- at = statements.get(u.line)- if at is not None:- observed = frozenset(- defs[i].line for i in definitions_at(def_use, reaching, at[0], u.name, at[1])- )- else:- k = bisect_left(dl, u.line)- observed = frozenset(dl[k - 1 : k])- reads[u.name].append((u.line, observed))- return reads---def _statements_by_line(cfg: CFG) -> dict[int, tuple[int, int]]:- """Per line, (block id, start line) of the innermost recorded statement- covering it."""- best: dict[int, tuple[int, int, int]] = {}- for block in cfg.blocks:- for st in block.statements:- width = st.end_line - st.start_line- for line in range(st.start_line, st.end_line + 1):- held = best.get(line)- if held is None or width < held[0]:- best[line] = (width, block.id, st.start_line)- return {line: (bid, start) for line, (_w, bid, start) in best.items()}---def _infer_in_out(- def_lines: dict[str, list[int]],- use_lines: dict[str, list[int]],- s: int,- e: int,- declared_first: dict[str, frozenset[int]],- *,- reads: dict[str, list[tuple[int, frozenset[int]]]],- valueless: dict[str, frozenset[int]] | None = None,-) -> tuple[tuple[str, ...], tuple[str, ...]]:- """Infer IN (parameters) and OUT (return) variables for span ``[s, e]``.-- IN: a variable read in the span whose first in-span read is not preceded by- an in-span write, and which has a definition before the span (a parameter or- an earlier assignment). A write on the same line as that read precedes it- only where *declared_first* (:func:`_declared_before_read`) says the line- declares the name first. An earlier in-span write covers a later read only- on the paths that run it, so a variable is also IN when some in-span read- of it may still observe a definition from before the span (*reads*,- :func:`_reads_observing`): the write sits under an ``if``, in a loop- body, or in an arm. A declaration that gives the name no value- (*valueless*, :func:`_valueless_lines`) is not such a definition: the- compiler has proved the read assigned where the may-analysis cannot, and- a call that passed the name would not compile. OUT: a variable written in- the span with a read after it that may observe one of those writes- (*reads* again). A read before the span that a write in it reaches around- a loop is not an OUT: :func:`_loop_carry_free` refuses those spans. Both- line lists are sorted (:func:`_var_lines`), so the span tests are bisects.- """- params: list[str] = []- returns: list[str] = []- for var in sorted(set(def_lines) | set(use_lines)):- dl = def_lines.get(var, [])- ul = use_lines.get(var, [])- u_lo, u_hi = bisect_left(ul, s), bisect_right(ul, e)- d_lo, d_hi = bisect_left(dl, s), bisect_right(dl, e)-- if u_lo < u_hi and d_lo > 0:- first_use = ul[u_lo]- declaring = declared_first.get(var, ())- bare = valueless.get(var, ()) if valueless else ()- uncovered = first_use not in declaring and not (d_lo < d_hi and dl[d_lo] < first_use)- if uncovered or any(- s <= ln <= e- and ln not in declaring- and any(d < s and d not in bare for d in seen)- for ln, seen in reads.get(var, ())- ):- params.append(var)-- if d_lo < d_hi and any(- ln > e and any(s <= d <= e for d in seen) for ln, seen in reads.get(var, ())- ):- returns.append(var)- return tuple(params), tuple(returns)---class _SharedState(NamedTuple):- """Per variable a closure shares with the function, sorted line lists:- the closures' reads and writes (``Captured.shared`` / ``.writes``), the- function's own defs, all its references, and its binding declarations."""-- reads: dict[str, list[int]]- writes: dict[str, list[int]]- defs: dict[str, list[int]]- refs: dict[str, list[int]]- binds: dict[str, frozenset[int]]---def _closure_state(- def_use: FunctionDefUse,- def_lines: dict[str, list[int]],- use_lines: dict[str, list[int]],-) -> _SharedState | None:- """The closure-shared locals of a function, or None when it has none- (then the gate costs nothing per span)."""- reads: dict[str, list[int]] = defaultdict(list)- writes: dict[str, list[int]] = defaultdict(list)- for out, occurrences in (- (reads, def_use.captured.shared),- (writes, def_use.captured.writes),- ):- for occ in occurrences:- if occ.name in def_lines: # a name this function binds- out[occ.name].append(occ.line)- if not reads and not writes:- return None- names = set(reads) | set(writes)- return _SharedState(- reads={var: sorted(lines) for var, lines in reads.items()},- writes={var: sorted(lines) for var, lines in writes.items()},- defs={var: def_lines.get(var, []) for var in names},- refs={var: sorted((*def_lines.get(var, ()), *use_lines.get(var, ()))) for var in names},- binds=_declaration_lines(def_use, binding=True),- )---def _count_in(lines: list[int], s: int, e: int) -> int:- return bisect_right(lines, e) - bisect_left(lines, s)---def _closure_state_crosses(- st: _SharedState, span: list[Node], s: int, e: int, block: Node, lmap: LanguageNodeMap-) -> bool:- """True when a local a closure shares crosses the span boundary.-- A closure runs when it is called, not where it is written, so line- liveness cannot place its reads and writes; an IN/OUT signature copies a- value where the closure needs the variable itself. Refused:-- - a closure in the span writes a local anything outside the span refers- to (code, or a closure there): the lifted closure writes the helper's- copy. A counter declared in the span with its ``inc`` closure is- refused this way when the count is read after the span;- - a closure outside the span writes a local the span refers to: the span- reads or writes a stale copy;- - the span writes a local that a closure written above it reads: the- closure runs later and sees the old value. Measured on hermes- ``apps/desktop/src/lib/ansi.ts::parseAnsi``, whose ``pushText`` reads- the ``bold`` / ``fg`` a span inside the loop sets.-- The last two are waived when the span's name is a binding of its own- (:func:`_span_binds`, :func:`_span_only_rebinds`): the closure's variable- is then another one.- """- for var in st.reads.keys() | st.writes.keys():- writes = st.writes.get(var, [])- reads = st.reads.get(var, [])- refs = st.refs[var]- w_in = _count_in(writes, s, e)- if w_in and (- w_in < len(writes)- or _count_in(reads, s, e) < len(reads)- or _count_in(refs, s, e) < len(refs)- ):- return True- stale = w_in < len(writes) and _count_in(refs, s, e) > 0- if stale and not _span_binds(st, var, span, block, lmap):- return True- overwritten = bool(reads) and reads[0] < s and _count_in(st.defs[var], s, e) > 0- if overwritten and not _span_only_rebinds(st, var, s, e, block, lmap):- return True- return False---def _declares_in(node: Node, lines: frozenset[int], lmap: LanguageNodeMap) -> bool:- """True when *node* is a declaration statement holding one of *lines*."""- lo, hi = node.start_point[0] + 1, node.end_point[0] + 1- return node.type in lmap.local_decl_kinds and any(lo <= ln <= hi for ln in lines)---def _declared_above(binds: frozenset[int], block: Node, s: int, lmap: LanguageNodeMap) -> bool:- """True when a statement of *block* above the span already declares the- name: a re-declaration in the same scope (Go's ``v, err := h()``) is an- assignment to it, not a new binding."""- return any(- node.end_point[0] + 1 < s and _declares_in(node, binds, lmap)- for node in block.named_children- )---def _span_binds(- st: _SharedState, var: str, span: list[Node], block: Node, lmap: LanguageNodeMap-) -> bool:- """True when one of the span's own top-level statements declares a new- binding of *var*, so every reference to it in the span is to that one.-- Top-level only: a declaration in a nested block shadows just that block.- """- binds = st.binds.get(var, frozenset())- s = span[0].start_point[0] + 1- return (- bool(binds)- and not _declared_above(binds, block, s, lmap)- and any(_declares_in(node, binds, lmap) for node in span)- )---def _span_only_rebinds(- st: _SharedState, var: str, s: int, e: int, block: Node, lmap: LanguageNodeMap-) -> bool:- """True when every write the span makes to *var* creates a new binding- (a declaration or a loop binder, at any depth), so none of them reaches- the variable a closure outside the span reads."""- binds = st.binds.get(var, frozenset())- defs = st.defs[var][bisect_left(st.defs[var], s) : bisect_right(st.defs[var], e)]- return all(ln in binds for ln in defs) and not _declared_above(binds, block, s, lmap)---def _holds_a_named_nested_function(span: list[Node], lmap: LanguageNodeMap) -> bool:- """True when the span contains a nested function that binds its own name.-- Def/use is computed per function and deliberately does not descend into a- nested scope, so a sibling closure's call to such a helper is invisible- here. Lifting the declaration out of the scope moves the binding away from- those callers, and nothing in the facts would show it.-- A named nested function exists to be called from elsewhere in its scope, so- the safe answer is to refuse rather than to guess at its callers. An- anonymous lambda bound to a local is a value and stays governed by the- ordinary IN/OUT rules. Measured on this repo's ranked head:- ``vscode/src/features/changeIntel.ts::registerChangeIntel`` offered its whole- ``render`` declaration while a sibling closure called it.- """- kinds = lmap.function_kinds- if not kinds:- return False- for st in span:- stack = [st]- while stack:- node = stack.pop()- if node.type in kinds and node.child_by_field_name("name") is not None:- return True- stack.extend(node.children)- return False---def _hoisted_bindings(- def_lines: dict[str, list[int]],- use_lines: dict[str, list[int]],-) -> list[tuple[int, int]]:- """First-definition lines of names the function reads before defining them.-- A read before the only definition of a name can only work by hoisting: a- JS/TS ``function foo()`` declaration is visible from the top of its scope,- so code above it calls it. A span holding that definition cannot be lifted -- an OUT cannot express it, because the value has to exist *before* the span- runs, not after - so :func:`find_extractions` refuses any span containing- one of these lines.-- Returns ``(first_def, first_use)`` pairs. A span refuses when its range holds- a ``first_def`` whose ``first_use`` sits *above* the span: the earlier read- has no earlier definition to answer it, so the span holds the only one.- A read that sits inside the span, before the definition, is a different- shape and is left to the loop-carried gate.-- Computed once per function because the shape is rare (usually no name- qualifies at all), and asking per candidate span meant walking every- variable thousands of times. Both lists are sorted by :func:`_var_lines`,- so the first entry of each is the earliest.-- Measured on this repo's ranked head:- ``vscode/src/features/changeIntel.ts::registerChangeIntel`` offered its whole- ``render`` declaration as a span while ``render(partners)`` sat above it.- """- hoisted: list[tuple[int, int]] = []- for var, defs in def_lines.items():- uses = use_lines.get(var)- if defs and uses and uses[0] < defs[0]:- hoisted.append((defs[0], uses[0]))- return sorted(hoisted)---def _declaration_lines(- def_use: FunctionDefUse, *, binding: bool = False-) -> dict[str, frozenset[int]]:- """Per variable, the lines that declare it (a ``let`` / ``var`` / ``:=`` /- typed local, as the dialect marks with ``declared_at``, or an import).- With *binding*, the lines that create a new binding instead- (``declares``: multi-line declarators and loop binders included, a TS/JS- ``var`` not)."""- lines: dict[str, set[int]] = defaultdict(set)- for d in def_use.definitions:- if (d.declares if binding else d.declared_at is not None or d.imports):- lines[d.var].add(d.line)- return {var: frozenset(found) for var, found in lines.items()}---def _declaration_escapes(- s: int,- e: int,- returns: tuple[str, ...],- decl_lines: dict[str, frozenset[int]],- def_lines: dict[str, list[int]],- use_lines: dict[str, list[int]],- declared_first: dict[str, frozenset[int]],-) -> bool:- """True when the span declares a name the code after it still refers to.-- A returned name is fine: the caller declares it from the helper's result.- Otherwise the declaration leaves with the span and the caller's next- reference names nothing, even when that reference is a plain assignment- (Go ``var out T`` in the span, ``out = x`` after it, which is why liveness- did not make it an OUT). A later reference that declares the name afresh- (``y, err := g()``, a second ``for i := ...``, whose reads on that line- follow the declaration per *declared_first*) needs nothing from the span.- Block scopes are not modelled, so a same-named variable of an outer scope- read after the span also refuses: a missed span, never a broken one.- """- return any(- var not in returns- and any(s <= ln <= e for ln in declared)- and _needed_after(var, e, declared, def_lines, use_lines, declared_first)- for var, declared in decl_lines.items()- )---def _bindings_escape(- s: int,- e: int,- returns: tuple[str, ...],- decl_lines: dict[str, frozenset[int]],- def_lines: dict[str, list[int]],- use_lines: dict[str, list[int]],- declared_first: dict[str, frozenset[int]],- outer: dict[str, frozenset[int]],-) -> bool:- """A binding the span cannot take along: a declaration the code after it- still needs (:func:`_declaration_escapes`), or a ``global`` / ``nonlocal``- write whose declaration stays behind (:func:`_outer_write_escapes`)."""- return _declaration_escapes(- s, e, returns, decl_lines, def_lines, use_lines, declared_first- ) or _outer_write_escapes(s, e, outer, def_lines)---def _outer_write_escapes(- s: int, e: int, outer: dict[str, frozenset[int]], def_lines: dict[str, list[int]]-) -> bool:- """True when the span writes a ``global`` / ``nonlocal`` name it cannot- take its declaration along for: one declared outside the span, or also- written outside it. In a helper without the declaration the write binds a- local and the module value never changes."""- for var, declared in outer.items():- writes = def_lines.get(var, ())- if not any(s <= ln <= e for ln in writes):- continue- if any(not s <= ln <= e for ln in (*declared, *writes)):- return True- return False---def _needed_after(- var: str,- e: int,- declared: frozenset[int],- def_lines: dict[str, list[int]],- use_lines: dict[str, list[int]],- declared_first: dict[str, frozenset[int]],-) -> bool:- """True when the first reference to *var* after line *e* relies on an- earlier declaration: it is a plain use or assignment, not a fresh one."""- uses = use_lines.get(var, ())- after = [ln for ln in (*def_lines.get(var, ()), *uses) if ln > e]- if not after:- return False- first = min(after)- if first not in declared:- return True- return first in uses and first not in declared_first.get(var, ())---def _outs_definitely_assigned(- span: list[Node],- returns: tuple[str, ...],- def_lines: dict[str, list[int]],- lmap: LanguageNodeMap,-) -> bool:- """True when every OUT variable is written on *every* path through the span.-- A conditionally written OUT is the unsound case line-based liveness cannot- see: the helper returns the unwritten value on the paths that skip the- write, and the caller's assignment then clobbers the live one. The proof- needs branch structure, so it runs structurally here and the span is- refused whenever the proof does not go through.- """- return all(_stmts_assign(span, var, def_lines.get(var, []), lmap) for var in returns)---def _stmts_assign(stmts: list[Node], var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:- """True when *stmts* writes *var* on every path through them."""- return any(_stmt_assigns(st, var, defs, lmap) for st in stmts)---def _stmt_assigns(st: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:- """True only when *st* is proved to write *var* however it is entered.-- Positive proof only: a statement kind this cannot classify returns False.- Defaulting to True would let any statement sharing a line with a- conditional write stand in as the proof.- """- lo = st.start_point[0] + 1- hi = st.end_point[0] + 1- if not any(lo <= d <= hi for d in defs):- return False- if _is_conditional(st, lmap):- return _conditional_assigns(st, var, defs, lmap)- if st.type in lmap.block_kinds:- container = _unwrap_container(st, lmap.block_kinds)- return _stmts_assign(container.named_children, var, defs, lmap)- if st.type in lmap.with_kinds or st.type in lmap.statement_wrapper_kinds:- return _stmts_assign(list(st.named_children), var, defs, lmap)- return _unconditional_write(st, var, lmap)---def _unconditional_write(st: Node, var: str, lmap: LanguageNodeMap) -> bool:- """True when *st* contains a write to *var* no branch or loop guards.-- The walk stops at any control container (a loop body may run zero times, a- branch arm may not be taken, a catch arm may not fire) and at nested- scopes, so only writes on the statement's own straight-line path count.- """- write_kinds = lmap.assignment_kinds | lmap.augmented_assign_kinds | lmap.local_decl_kinds- if not write_kinds:- return False- barriers = (- lmap.loop_kinds- | lmap.try_kinds- | lmap.catch_kinds- | lmap.switch_kinds- | lmap.case_kinds- | lmap.branch_kinds- | lmap.if_kinds- | lmap.function_kinds- | lmap.lambda_kinds- )- if st.type in barriers:- return False- stack = [st]- while stack:- node = stack.pop()- if node.id != st.id and node.type in barriers:- continue- if node.type in write_kinds and _writes_var(node, var):- return True- stack.extend(node.children)- return False---def _writes_var(node: Node, var: str) -> bool:- """True when *var* is on the binding side of this write node."""- for field in ("left", "pattern", "declarator", "name"):- target = node.child_by_field_name(field)- if target is not None:- return var in _identifiers(target)- return var in _identifiers(node)---def _is_conditional(node: Node, lmap: LanguageNodeMap) -> bool:- return node.type in lmap.if_kinds or (- node.type in lmap.branch_kinds and node.child_by_field_name("condition") is not None- )---def _conditional_assigns(node: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:- """True when a conditional chain is exhaustive and every arm writes *var*.-- Two chain shapes. Python flattens ``elif``/``else`` into repeated- ``alternative`` fields on one ``if``, so the terminal else is a sibling of- the chained arms. The C family nests instead: the single ``alternative`` is- another ``if``, whose own else ends the chain. Both must be exhaustive.- """- consequence = node.child_by_field_name("consequence") or node.child_by_field_name("body")- if consequence is None:- return False- alternatives = list(node.children_by_field_name("alternative"))- if not alternatives:- return False # no else at all: some path skips the write- if not _branch_assigns(consequence, var, defs, lmap):- return False- for alt in alternatives:- if _is_conditional(alt, lmap):- # Nested chain (C family): it carries the chain's terminal else.- if len(alternatives) == 1:- return _conditional_assigns(alt, var, defs, lmap)- # Flattened chain (Python's ``elif``): the else is a sibling below.- arm = alt.child_by_field_name("consequence") or alt.child_by_field_name("body")- if arm is None or not _branch_assigns(arm, var, defs, lmap):- return False- elif not _branch_assigns(alt, var, defs, lmap):- return False- return any(not _is_conditional(alt, lmap) for alt in alternatives)---def _branch_assigns(node: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:- if _is_conditional(node, lmap):- return _conditional_assigns(node, var, defs, lmap)- if node.type in lmap.block_kinds:- container = _unwrap_container(node, lmap.block_kinds)- return _stmts_assign(container.named_children, var, defs, lmap)- return _stmts_assign(list(node.named_children), var, defs, lmap)---def _rebinding_defs_dropped(- analysis: FunctionAnalysis, def_lines: dict[str, list[int]], lmap: LanguageNodeMap-) -> dict[str, list[int]]:- """*def_lines* without the names exception handlers bind on their head- line (``except E as e:``, ``catch (err)``). Such a binding never reaches- the next iteration: Python unbinds it at the handler's end and a catch- parameter is scoped to its block, so a read of the name elsewhere in the- loop is another handler's own."""- heads = [- st.node- for block in analysis.cfg.blocks- for st in block.statements- if st.is_head and st.kind in lmap.catch_kinds and st.node is not None- ]- if not heads:- return def_lines- bound: set[tuple[str, int]] = set()- for head in heads:- # Only the clause's own binder: before its body, so a one-line body's- # write on the same line still counts.- body = next((c for c in head.named_children if c.type in lmap.block_kinds), None)- end = body.start_point if body is not None else head.end_point- bound |= {- (d.var, d.line)- for d in analysis.def_use.definitions- if head.start_point <= (d.line - 1, d.column) < end- }- return {var: [ln for ln in lines if (var, ln) not in bound] for var, lines in def_lines.items()}---def _loop_carry_free(- span: list[Node],- loop: Node,- s: int,- e: int,- def_lines: dict[str, list[int]],- use_lines: dict[str, list[int]],- lmap: LanguageNodeMap,-) -> bool:- """True when a span nested in *loop* carries no state between iterations.-- Two shapes are refused. A variable the span writes that the loop also reads- without the span having written it first is loop-carried: that read takes- the previous iteration's value, and lifting the write into a helper whose- result is discarded silently drops it.-- The read does not have to sit above the span. ``clojure.py::_spec_namespaces``- reads ``expect_ns`` and then writes it, both inside one candidate span, in a- ``while`` loop: textually the read precedes the write, so nothing follows- the span to make the variable an OUT, and the state machine breaks on the- next iteration. Testing only for reads above the span missed it, so the test- is "read with no in-span write before it", which subsumes the old one.-- Also refused: a call on a name the loop header reads, which mutates the- state the loop iterates over (``entries.remove(entry)`` inside- ``for entry in entries``), and which an IN/OUT signature cannot express.- """- loop_start = loop.start_point[0] + 1- for var, lines in def_lines.items():- in_span_writes = [ln for ln in lines if s <= ln <= e]- if not in_span_writes:- continue- for read in use_lines.get(var, []):- if not loop_start <= read <= e:- continue- if not any(write < read for write in in_span_writes):- return False-- carried = _loop_header_names(loop, lmap)- if not carried:- return True- scope_kinds = lmap.function_kinds | lmap.lambda_kinds- for st in span:- for call in _descend(st, lmap.call_kinds, scope_kinds):- callee = call.child_by_field_name("function")- if callee is None:- named = call.named_children- callee = named[0] if named else None- root = _receiver_root(callee) if callee is not None else None- if root is not None and root in carried:- return False- return True---def _receiver_root(node: Node) -> str | None:- """The base name a call is made on: ``entries`` in ``entries.remove(x)``.-- Only the root counts. Matching any name in the callee read- ``existing.pages.push(page)`` as touching the ``pages`` the loop iterates,- when the receiver is ``existing`` and the collision is in a member name.- """- cur = node- while True:- nxt = None- for field in ("object", "operand", "value", "argument", "function"):- child = cur.child_by_field_name(field)- if child is not None:- nxt = child- break- if nxt is None:- break- cur = nxt- if not cur.children and cur.type.endswith("identifier"):- text = cur.text- return text.decode("utf-8", "replace") if text else None- return None---# Fields a loop header binds through. Read structurally: a line-range test-# cannot tell the header's binder from a variable reassigned on the body's-# first line, and getting that wrong drops the iterated collection out of the-# carried set, which is exactly the name the mutation check exists to catch.-_BINDER_FIELDS = ("left", "pattern", "declarator", "name")---def _loop_header_names(loop: Node, lmap: LanguageNodeMap) -> set[str]:- """Names the loop header reads, minus the ones it binds (the loop variable)."""- body = loop.child_by_field_name("body")- names: set[str] = set()- # The binder is a field of the loop itself in most grammars, and of a- # header clause it wraps in Go's ``range``.- bound: set[str] = _binder_names(loop)- for child in loop.children:- if body is not None and child.id == body.id:- continue- names |= _identifiers(child)- bound |= _binder_names(child)- return names - bound---def _binder_names(node: Node) -> set[str]:- """Identifiers in the binding position of *node* or of a header clause it- wraps (Go nests its range clause inside the ``for``)."""- out: set[str] = set()- for field in _BINDER_FIELDS:- target = node.child_by_field_name(field)- if target is not None:- out |= _identifiers(target)- if not out:- for child in node.named_children:- for field in _BINDER_FIELDS:- target = child.child_by_field_name(field)- if target is not None:- out |= _identifiers(target)- return out---def _identifiers(node: Node) -> set[str]:- out: set[str] = set()- stack = [node]- while stack:- cur = stack.pop()- if not cur.children and cur.type.endswith("identifier"):- text = cur.text- if text:- out.add(text.decode("utf-8", "replace"))- stack.extend(cur.children)- return out---def _descend(root: Node, kinds: frozenset[str], skip: frozenset[str]) -> list[Node]:- found: list[Node] = []- stack = [root]- while stack:- cur = stack.pop()- if cur.type in kinds:- found.append(cur)- for child in cur.children:- if child.type in skip:- continue- stack.append(child)- return found---def _unwrap_container(node: Node, block_kinds: frozenset[str]) -> Node:- """Descend through a single nested statement-container (Go's- ``block`` -> ``statement_list``) to the node whose named children are the- actual statements; returns *node* unchanged when it is already that node."""- cur = node- while True:- named = cur.named_children- if len(named) == 1 and named[0].type in block_kinds:- cur = named[0]- else:- return cur---def all_blocks(- fn_node: Node,- block_kinds: frozenset[str],- scope_kinds: frozenset[str],- loop_kinds: frozenset[str],-) -> list[tuple[Node, Node | None]]:- """Every statement container in the function (body + nested), excluding the- bodies of nested functions / lambdas, each paired with the innermost loop- enclosing it (``None`` at loop-free depth)."""- body = fn_node.child_by_field_name("body")- if body is None:- return []- blocks: list[tuple[Node, Node | None]] = []- stack: list[tuple[Node, Node | None]] = [(body, None)]- while stack:- node, loop = stack.pop()- if node.type in block_kinds:- blocks.append((node, loop))- is_loop = node.type in loop_kinds- body = node.child_by_field_name("body") if is_loop else None- for child in node.children:- if child.type in scope_kinds:- continue- # Only the loop's body repeats. A ``for ... else`` clause runs once.- in_loop = node if (is_loop and body is not None and child.id == body.id) else loop- stack.append((child, in_loop))- return blocks---def _exit_macros(lmap: LanguageNodeMap) -> tuple[frozenset[str], frozenset[str]]:- """The macro node kinds and the macro names that exit the function."""- return lmap.exit_macro_kinds, lmap.exit_macro_names---def _awaits(lmap: LanguageNodeMap) -> tuple[frozenset[str], frozenset[str]]:- """The await tokens, and the nodes that own the awaits inside them."""- return lmap.await_kinds, lmap.await_scope_kinds--def _span_kinds(scan: _Scan) -> FactKinds:await_kinds, await_scope_kinds = scan.awaitsreturn FactKinds(@@ -1495,95 +636,3 @@ def _push_children(t = child.typeif t not in scan.scopes or (scan.receiver is not None and t in scan.lambdas):stack.append((child, scope))---# Index access on a receiver field (``self.cache[k] = v``) assigns into it.-_INDEX_KINDS = frozenset({"subscript", "subscript_expression", "index_expression", "array_access"})-# Unpacking targets whose named children are targets themselves-# (``a, self.b = ...``, ``[this.x, y] = ...``).-_UNPACK_KINDS = frozenset(- {- "pattern_list",- "tuple_pattern",- "list_pattern",- "expression_list",- "tuple",- "list",- "array_pattern",- "object_pattern",- "parenthesized_expression",- "list_splat_pattern",- "rest_pattern",- }-)-# Marks a target rooted at the receiver whose shape is not read, so the-# span's assigned fields are unknown.-_UNREAD_TARGET = ""---def _receiver_ref(node: Node, receiver: Receiver, assigns: set[str]) -> bool:- """True when *node* names the receiver; an assignment to one of its- fields adds the field's name to *assigns* (the receiver leaf under it is- visited next)."""- if node.type in receiver.write_kinds:- target = (- node.child_by_field_name("left")- or node.child_by_field_name("argument")- or (node.named_children[0] if node.named_children else None)- )- if target is not None:- _collect_assigned(target, receiver, assigns)- return False- return _is_receiver(node, receiver)---def _collect_assigned(target: Node, receiver: Receiver, assigns: set[str]) -> None:- """The receiver fields one assignment target writes, unpacking included;- a target of another shape that mentions the receiver makes them unknown."""- if target.type in _UNPACK_KINDS:- for each in target.named_children:- _collect_assigned(each, receiver, assigns)- return- if target.type in ("assignment_pattern", "object_assignment_pattern"): # ``[this.a = 1]``- left = target.child_by_field_name("left")- if left is not None:- _collect_assigned(left, receiver, assigns)- return- if target.type == "pair_pattern": # ``{key: this.a}``- value = target.child_by_field_name("value")- if value is not None:- _collect_assigned(value, receiver, assigns)- return- field = _receiver_field(target, receiver)- if field:- assigns.add(field)- elif target.type not in receiver.access_kinds | _INDEX_KINDS and mentions_receiver(- target, receiver.names- ):- assigns.add(_UNREAD_TARGET)---def _is_receiver(node: Node, receiver: Receiver) -> bool:- """A leaf naming the receiver: a ``this`` / ``self`` / ``super`` token,- or an identifier spelling a named receiver (Python ``self``, Go ``s``)- or ``super`` (Python's no-argument ``super()``)."""- if node.children or not node.text:- return False- if node.type in receiver.names or node.type == SUPER:- return True- if node.type != "identifier":- return False- text = node.text.decode("utf-8", "replace")- return text in receiver.names or text == SUPER---def _receiver_field(target: Node, receiver: Receiver) -> str | None:- """``x`` for an assignment target ``self.x``, ``self.x.y`` or ``self.x[k]``."""- cur: Node | None = target- while cur is not None and (cur.type in receiver.access_kinds or cur.type in _INDEX_KINDS):- if cur.type in receiver.access_kinds:- field = self_member_name(cur, receiver.names)- if field is not None:- return field- cur = member_object(cur)- return Nonediff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice_liveness.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice_liveness.pynew file mode 100644index 000000000..bb67e403a--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice_liveness.py@@ -0,0 +1,350 @@+"""Line-based liveness over def/use facts: a span's IN / OUT and the bindings it cannot take along."""++from __future__ import annotations++from bisect import bisect_left, bisect_right+from collections import defaultdict+from typing import TYPE_CHECKING++from .reaching import definitions_at, observed_definitions++if TYPE_CHECKING:+ from collections.abc import Container++ from .analyze import FunctionAnalysis+ from .cfg import CFG+ from .defuse import FunctionDefUse+++def _var_lines(def_use: FunctionDefUse) -> tuple[dict[str, list[int]], dict[str, list[int]]]:+ """Per-variable sorted def lines and use lines from the def/use facts.++ Parameter definitions are included (seeded at the signature line), so a+ parameter naturally counts as "defined before" any body span. Reads inside+ nested closures (``def_use.captured.reads``) count as uses at their own line.+ An import is not a def here: a helper re-imports a name rather than take+ or return it, and :func:`_declaration_escapes` refuses a span holding an+ import that later code still reads.+ """+ def_lines: dict[str, list[int]] = defaultdict(list)+ use_lines: dict[str, list[int]] = defaultdict(list)+ imported: set[str] = set()+ for d in def_use.definitions:+ if d.imports:+ imported.add(d.var)+ else:+ def_lines[d.var].append(d.line)+ for bdu in def_use.blocks.values():+ for u in bdu.uses:+ # A may-def's paired use is bookkeeping, not a read: counted, a+ # binder declared inside a ``match`` arm became its own parameter.+ if not u.echo:+ use_lines[u.name].append(u.line)+ # A closure's read counts where the closure is written: lifting the code+ # around it moves the read with it. Only names this function binds matter.+ for u in def_use.captured.reads:+ if u.name in def_lines or u.name in imported:+ use_lines[u.name].append(u.line)+ for lines in def_lines.values():+ lines.sort()+ for lines in use_lines.values():+ lines.sort()+ return def_lines, use_lines+++def _import_reads(analysis: FunctionAnalysis) -> tuple[tuple[int, int], ...]:+ """Function-local import lines paired with the reads they reach."""+ imports = {+ (definition.var, definition.line)+ for definition in analysis.reaching.definitions+ if definition.imports+ }+ if not imports:+ return ()+ import_lines: dict[str, list[int]] = defaultdict(list)+ for name, line in sorted(imports):+ import_lines[name].append(line)+ return tuple(+ (definition_line, use_line)+ for name, reads in _reads_observing(analysis, import_lines).items()+ for use_line, seen in reads+ for definition_line in seen+ if (name, definition_line) in imports+ )+++def _declared_before_read(def_use: FunctionDefUse) -> dict[str, frozenset[int]]:+ """Per variable, the lines where a declaration of it comes before its first+ read on the same line.++ Lines alone cannot order a write and a read that share one. A C-style+ ``for (int i = 0; i < n; i++)`` declares ``i`` and then reads it, while+ ``total = total + a[i]`` reads ``total`` and then writes it. Only a+ declaration is ordered here, by where its declarator ends: a read past that+ point sees the new name, and a read inside the declaration's own+ initializer (Go's ``x := x + 1`` in an inner scope) still sees the outer+ one. A plain assignment keeps the line rule.+ """+ first_read: dict[tuple[str, int], int] = {}+ for bdu in def_use.blocks.values():+ for u in bdu.uses:+ if u.echo:+ continue+ key = (u.name, u.line)+ first_read[key] = min(u.column, first_read.get(key, u.column))+ declared: dict[str, set[int]] = defaultdict(set)+ for d in def_use.definitions:+ # No read on the line sorts before any declarator, so it never matches.+ if d.declared_at is not None and d.declared_at <= first_read.get((d.var, d.line), -1):+ declared[d.var].add(d.line)+ return {var: frozenset(lines) for var, lines in declared.items()}+++def _valueless_lines(def_use: FunctionDefUse) -> dict[str, frozenset[int]]:+ """Per variable, the lines whose only definitions of it are declarations+ that give it no value (``Definition.valueless``): Java's ``List<T> xs;``,+ TypeScript's ``let x: T;``. A line that also assigns the name+ (``int x; x = 1;``) is not one: what a read sees from it is the value."""+ bare: dict[str, set[int]] = defaultdict(set)+ valued: dict[str, set[int]] = defaultdict(set)+ for d in def_use.definitions:+ (bare if d.valueless else valued)[d.var].add(d.line)+ return {var: frozenset(at - valued[var]) for var, at in bare.items()}+++def _reads_observing(+ analysis: FunctionAnalysis, def_lines: dict[str, list[int]]+) -> dict[str, list[tuple[int, frozenset[int]]]]:+ """Per variable, each read's line with the def lines it may observe.++ Reads take reaching definitions over the CFG, so a later write on a path+ that need not run (an ``else`` arm, a ``try`` body that may raise first)+ does not hide the span's value from the read. A closure's read observes+ what reaches the statement that creates the closure, or, where no recorded+ statement covers it, the nearest def above it: a read is never dropped,+ since a dropped read can only drop a return. Only names in *def_lines*+ count for closures, as in :func:`_var_lines`.+ """+ def_use, reaching = analysis.def_use, analysis.reaching+ defs = reaching.definitions+ reads: dict[str, list[tuple[int, frozenset[int]]]] = defaultdict(list)+ for use, seen in observed_definitions(def_use, reaching):+ if not use.echo:+ reads[use.name].append((use.line, frozenset(defs[i].line for i in seen)))+ statements: dict[int, tuple[int, int]] | None = None+ for u in def_use.captured.reads:+ dl = def_lines.get(u.name)+ if not dl:+ continue+ if statements is None:+ statements = _statements_by_line(analysis.cfg)+ at = statements.get(u.line)+ if at is not None:+ observed = frozenset(+ defs[i].line for i in definitions_at(def_use, reaching, at[0], u.name, at[1])+ )+ else:+ k = bisect_left(dl, u.line)+ observed = frozenset(dl[k - 1 : k])+ reads[u.name].append((u.line, observed))+ return reads+++def _statements_by_line(cfg: CFG) -> dict[int, tuple[int, int]]:+ """Per line, (block id, start line) of the innermost recorded statement+ covering it."""+ best: dict[int, tuple[int, int, int]] = {}+ for block in cfg.blocks:+ for st in block.statements:+ width = st.end_line - st.start_line+ for line in range(st.start_line, st.end_line + 1):+ held = best.get(line)+ if held is None or width < held[0]:+ best[line] = (width, block.id, st.start_line)+ return {line: (bid, start) for line, (_w, bid, start) in best.items()}+++def _infer_in_out(+ def_lines: dict[str, list[int]],+ use_lines: dict[str, list[int]],+ s: int,+ e: int,+ declared_first: dict[str, frozenset[int]],+ *,+ reads: dict[str, list[tuple[int, frozenset[int]]]],+ valueless: dict[str, frozenset[int]] | None = None,+) -> tuple[tuple[str, ...], tuple[str, ...]]:+ """Infer IN (parameters) and OUT (return) variables for span ``[s, e]``.++ IN: a variable read in the span whose first in-span read is not preceded by+ an in-span write, and which has a definition before the span (a parameter or+ an earlier assignment). A write on the same line as that read precedes it+ only where *declared_first* (:func:`_declared_before_read`) says the line+ declares the name first. An earlier in-span write covers a later read only+ on the paths that run it, so a variable is also IN when some in-span read+ of it may still observe a definition from before the span (*reads*,+ :func:`_reads_observing`): the write sits under an ``if``, in a loop+ body, or in an arm. A declaration that gives the name no value+ (*valueless*, :func:`_valueless_lines`) is not such a definition: the+ compiler has proved the read assigned where the may-analysis cannot, and+ a call that passed the name would not compile. OUT: a variable written in+ the span with a read after it that may observe one of those writes+ (*reads* again). A read before the span that a write in it reaches around+ a loop is not an OUT: :func:`_loop_carry_free` refuses those spans. Both+ line lists are sorted (:func:`_var_lines`), so the span tests are bisects.+ """+ params: list[str] = []+ returns: list[str] = []+ for var in sorted(set(def_lines) | set(use_lines)):+ dl = def_lines.get(var, [])+ ul = use_lines.get(var, [])+ u_lo, u_hi = bisect_left(ul, s), bisect_right(ul, e)+ d_lo, d_hi = bisect_left(dl, s), bisect_right(dl, e)++ if u_lo < u_hi and d_lo > 0:+ first_use = ul[u_lo]+ declaring = declared_first.get(var, ())+ covered = first_use in declaring or (d_lo < d_hi and dl[d_lo] < first_use)+ bare = valueless.get(var, ()) if valueless else ()+ if not covered or _sees_value_from_before(reads.get(var, ()), s, e, declaring, bare):+ params.append(var)++ if d_lo < d_hi and _span_write_seen_after(reads.get(var, ()), s, e):+ returns.append(var)+ return tuple(params), tuple(returns)+++def _sees_value_from_before(+ var_reads: list[tuple[int, frozenset[int]]],+ s: int,+ e: int,+ declaring: Container[int],+ bare: Container[int],+) -> bool:+ """Some read in ``[s, e]`` may observe a valued definition from before the span."""+ return any(+ s <= ln <= e and ln not in declaring and any(d < s and d not in bare for d in seen)+ for ln, seen in var_reads+ )+++def _span_write_seen_after(var_reads: list[tuple[int, frozenset[int]]], s: int, e: int) -> bool:+ """Some read after line *e* may observe a write inside ``[s, e]``."""+ return any(ln > e and any(s <= d <= e for d in seen) for ln, seen in var_reads)+++def _hoisted_bindings(+ def_lines: dict[str, list[int]],+ use_lines: dict[str, list[int]],+) -> list[tuple[int, int]]:+ """``(first_def, first_use)`` for names the function reads before defining them.++ Such a read works only by hoisting (a JS/TS ``function foo()``), and an OUT+ cannot express a value needed before the span runs, so a span holding a+ ``first_def`` whose ``first_use`` sits above it is refused. Computed once+ per function; both line lists are sorted, so their heads are the earliest.+ """+ hoisted: list[tuple[int, int]] = []+ for var, defs in def_lines.items():+ uses = use_lines.get(var)+ if defs and uses and uses[0] < defs[0]:+ hoisted.append((defs[0], uses[0]))+ return sorted(hoisted)+++def _declaration_lines(+ def_use: FunctionDefUse, *, binding: bool = False+) -> dict[str, frozenset[int]]:+ """Per variable, the lines that declare it (a ``let`` / ``var`` / ``:=`` /+ typed local, as the dialect marks with ``declared_at``, or an import).+ With *binding*, the lines that create a new binding instead+ (``declares``: multi-line declarators and loop binders included, a TS/JS+ ``var`` not)."""+ lines: dict[str, set[int]] = defaultdict(set)+ for d in def_use.definitions:+ if (d.declares if binding else d.declared_at is not None or d.imports):+ lines[d.var].add(d.line)+ return {var: frozenset(found) for var, found in lines.items()}+++def _declaration_escapes(+ s: int,+ e: int,+ returns: tuple[str, ...],+ decl_lines: dict[str, frozenset[int]],+ def_lines: dict[str, list[int]],+ use_lines: dict[str, list[int]],+ declared_first: dict[str, frozenset[int]],+) -> bool:+ """True when the span declares a name the code after it still refers to.++ A returned name is fine: the caller declares it from the helper's result.+ Otherwise the declaration leaves with the span and the caller's next+ reference names nothing, even when that reference is a plain assignment+ (Go ``var out T`` in the span, ``out = x`` after it, which is why liveness+ did not make it an OUT). A later reference that declares the name afresh+ (``y, err := g()``, a second ``for i := ...``, whose reads on that line+ follow the declaration per *declared_first*) needs nothing from the span.+ Block scopes are not modelled, so a same-named variable of an outer scope+ read after the span also refuses: a missed span, never a broken one.+ """+ return any(+ var not in returns+ and any(s <= ln <= e for ln in declared)+ and _needed_after(var, e, declared, def_lines, use_lines, declared_first)+ for var, declared in decl_lines.items()+ )+++def _bindings_escape(+ s: int,+ e: int,+ returns: tuple[str, ...],+ decl_lines: dict[str, frozenset[int]],+ def_lines: dict[str, list[int]],+ use_lines: dict[str, list[int]],+ declared_first: dict[str, frozenset[int]],+ outer: dict[str, frozenset[int]],+) -> bool:+ """A binding the span cannot take along: a declaration the code after it+ still needs (:func:`_declaration_escapes`), or a ``global`` / ``nonlocal``+ write whose declaration stays behind (:func:`_outer_write_escapes`)."""+ return _declaration_escapes(+ s, e, returns, decl_lines, def_lines, use_lines, declared_first+ ) or _outer_write_escapes(s, e, outer, def_lines)+++def _outer_write_escapes(+ s: int, e: int, outer: dict[str, frozenset[int]], def_lines: dict[str, list[int]]+) -> bool:+ """True when the span writes a ``global`` / ``nonlocal`` name it cannot+ take its declaration along for: one declared outside the span, or also+ written outside it. In a helper without the declaration the write binds a+ local and the module value never changes."""+ for var, declared in outer.items():+ writes = def_lines.get(var, ())+ if not any(s <= ln <= e for ln in writes):+ continue+ if any(not s <= ln <= e for ln in (*declared, *writes)):+ return True+ return False+++def _needed_after(+ var: str,+ e: int,+ declared: frozenset[int],+ def_lines: dict[str, list[int]],+ use_lines: dict[str, list[int]],+ declared_first: dict[str, frozenset[int]],+) -> bool:+ """True when the first reference to *var* after line *e* relies on an+ earlier declaration: it is a plain use or assignment, not a fresh one."""+ uses = use_lines.get(var, ())+ after = [ln for ln in (*def_lines.get(var, ()), *uses) if ln > e]+ if not after:+ return False+ first = min(after)+ if first not in declared:+ return True+ return first in uses and first not in declared_first.get(var, ())diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice_nodes.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice_nodes.pynew file mode 100644index 000000000..56b510046--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice_nodes.py@@ -0,0 +1,182 @@+"""Tree walkers the Extract Method slicer reads statement structure with."""++from __future__ import annotations++from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from tree_sitter import Node++ from ..complexity.languages import LanguageNodeMap+++def _function_lines(fn_node: Node) -> list[str]:+ """Source rows indexed by absolute row, rebuilt from the function's own text.++ Lets the walker's NLOC rule (``_code_line_numbers``) run here without the+ file's bytes; rows above the function are never read, so they stay empty.+ """+ text = (fn_node.text or b"").decode("utf-8", errors="replace")+ return [""] * fn_node.start_point[0] + text.splitlines()+++def _holds_a_named_nested_function(span: list[Node], lmap: LanguageNodeMap) -> bool:+ """True when the span contains a nested function that binds its own name.++ Def/use does not descend into nested scopes, so a sibling closure calling+ that helper is invisible; lifting the declaration would strand the call.+ An anonymous lambda bound to a local is a value and follows the IN/OUT rules.+ """+ kinds = lmap.function_kinds+ if not kinds:+ return False+ for st in span:+ stack = [st]+ while stack:+ node = stack.pop()+ if node.type in kinds and node.child_by_field_name("name") is not None:+ return True+ stack.extend(node.children)+ return False+++def _receiver_root(node: Node) -> str | None:+ """The base name a call is made on: ``entries`` in ``entries.remove(x)``.++ Only the root counts. Matching any name in the callee read+ ``existing.pages.push(page)`` as touching the ``pages`` the loop iterates,+ when the receiver is ``existing`` and the collision is in a member name.+ """+ cur = node+ while True:+ nxt = None+ for field in ("object", "operand", "value", "argument", "function"):+ child = cur.child_by_field_name(field)+ if child is not None:+ nxt = child+ break+ if nxt is None:+ break+ cur = nxt+ if not cur.children and cur.type.endswith("identifier"):+ text = cur.text+ return text.decode("utf-8", "replace") if text else None+ return None+++# Fields a loop header binds through. Read structurally: a line-range test+# cannot tell the header's binder from a variable reassigned on the body's+# first line, and getting that wrong drops the iterated collection out of the+# carried set, which is exactly the name the mutation check exists to catch.+_BINDER_FIELDS = ("left", "pattern", "declarator", "name")+++def _loop_header_names(loop: Node, lmap: LanguageNodeMap) -> set[str]:+ """Names the loop header reads, minus the ones it binds (the loop variable)."""+ body = loop.child_by_field_name("body")+ names: set[str] = set()+ # The binder is a field of the loop itself in most grammars, and of a+ # header clause it wraps in Go's ``range``.+ bound: set[str] = _binder_names(loop)+ for child in loop.children:+ if body is not None and child.id == body.id:+ continue+ names |= _identifiers(child)+ bound |= _binder_names(child)+ return names - bound+++def _binder_names(node: Node) -> set[str]:+ """Identifiers in the binding position of *node* or of a header clause it+ wraps (Go nests its range clause inside the ``for``)."""+ out: set[str] = set()+ for field in _BINDER_FIELDS:+ target = node.child_by_field_name(field)+ if target is not None:+ out |= _identifiers(target)+ if not out:+ for child in node.named_children:+ for field in _BINDER_FIELDS:+ target = child.child_by_field_name(field)+ if target is not None:+ out |= _identifiers(target)+ return out+++def _identifiers(node: Node) -> set[str]:+ out: set[str] = set()+ stack = [node]+ while stack:+ cur = stack.pop()+ if not cur.children and cur.type.endswith("identifier"):+ text = cur.text+ if text:+ out.add(text.decode("utf-8", "replace"))+ stack.extend(cur.children)+ return out+++def _descend(root: Node, kinds: frozenset[str], skip: frozenset[str]) -> list[Node]:+ found: list[Node] = []+ stack = [root]+ while stack:+ cur = stack.pop()+ if cur.type in kinds:+ found.append(cur)+ for child in cur.children:+ if child.type in skip:+ continue+ stack.append(child)+ return found+++def _unwrap_container(node: Node, block_kinds: frozenset[str]) -> Node:+ """Descend through a single nested statement-container (Go's+ ``block`` -> ``statement_list``) to the node whose named children are the+ actual statements; returns *node* unchanged when it is already that node."""+ cur = node+ while True:+ named = cur.named_children+ if len(named) == 1 and named[0].type in block_kinds:+ cur = named[0]+ else:+ return cur+++def all_blocks(+ fn_node: Node,+ block_kinds: frozenset[str],+ scope_kinds: frozenset[str],+ loop_kinds: frozenset[str],+) -> list[tuple[Node, Node | None]]:+ """Every statement container in the function (body + nested), excluding the+ bodies of nested functions / lambdas, each paired with the innermost loop+ enclosing it (``None`` at loop-free depth)."""+ body = fn_node.child_by_field_name("body")+ if body is None:+ return []+ blocks: list[tuple[Node, Node | None]] = []+ stack: list[tuple[Node, Node | None]] = [(body, None)]+ while stack:+ node, loop = stack.pop()+ if node.type in block_kinds:+ blocks.append((node, loop))+ is_loop = node.type in loop_kinds+ body = node.child_by_field_name("body") if is_loop else None+ for child in node.children:+ if child.type in scope_kinds:+ continue+ # Only the loop's body repeats. A ``for ... else`` clause runs once.+ in_loop = node if (is_loop and body is not None and child.id == body.id) else loop+ stack.append((child, in_loop))+ return blocks+++def _exit_macros(lmap: LanguageNodeMap) -> tuple[frozenset[str], frozenset[str]]:+ """The macro node kinds and the macro names that exit the function."""+ return lmap.exit_macro_kinds, lmap.exit_macro_names+++def _awaits(lmap: LanguageNodeMap) -> tuple[frozenset[str], frozenset[str]]:+ """The await tokens, and the nodes that own the awaits inside them."""+ return lmap.await_kinds, lmap.await_scope_kindsdiff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice_receiver.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice_receiver.pynew file mode 100644index 000000000..ccbff0d05--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice_receiver.py@@ -0,0 +1,252 @@+"""What a span or a whole function does with its receiver (``self`` / ``this``)."""++from __future__ import annotations++from dataclasses import dataclass+from typing import TYPE_CHECKING++from ..complexity.ast_utils import member_object, self_member_name+from ..complexity.body_facts import BodyTally, FactKinds, is_exit+from .dialects.base import SUPER, mentions_receiver+from .slice_nodes import _exit_macros++if TYPE_CHECKING:+ from tree_sitter import Node++ from ..complexity.languages import LanguageNodeMap+ from .defuse import FunctionDefUse+ from .dialects.base import Receiver+ from .slice import _Prefix+++@dataclass(frozen=True)+class FunctionFacts:+ """What a whole function does, counted on the complexity walk's visits.++ ``awaits``: it suspends outside any nested scope. ``is_generator``: it+ yields. ``uses_receiver`` / ``receiver_assigns`` as on :class:`Extraction`,+ for the whole body; ``receiver_assigns`` is None where an implicit receiver+ (Java, C++) can write fields by bare name, which only def/use can tell.+ ``early_exits``: returns and raises other than a final one.+ """++ awaits: bool+ is_generator: bool+ uses_receiver: bool | None+ receiver_assigns: tuple[str, ...] | None+ early_exits: int+++class _ReceiverSink:+ """``BodyTally.on_receiver``: whether the body names its receiver, and+ which of its fields it assigns."""++ __slots__ = ("assigns", "receiver", "uses")++ def __init__(self, receiver: Receiver) -> None:+ self.receiver = receiver+ self.uses = False+ self.assigns: set[str] = set()++ def __call__(self, node: Node) -> None:+ self.uses = _receiver_ref(node, self.receiver, self.assigns) or self.uses+++def body_tally(fn_node: Node, lmap: LanguageNodeMap, receiver: Receiver | None) -> BodyTally:+ """An empty tally for *fn_node*, for the complexity walk to fill. The+ receiver is watched only when the function's text names it."""+ watched = bool(+ receiver is not None and receiver.names and mentions_receiver(fn_node, receiver.names)+ )+ sink = _ReceiverSink(receiver) if watched and receiver is not None else None+ return BodyTally(_function_kinds(lmap, sink), sink)+++# Built once per language map and receiver shape: the walk asks per function.+_FUNCTION_KINDS: dict[tuple[int, frozenset[str]], FactKinds] = {}+++def _function_kinds(lmap: LanguageNodeMap, sink: _ReceiverSink | None) -> FactKinds:+ receiver = _receiver_kinds(sink.receiver) if sink else frozenset()+ key = (id(lmap), receiver)+ kinds = _FUNCTION_KINDS.get(key)+ if kinds is None:+ kinds = _FUNCTION_KINDS[key] = FactKinds(+ exits=lmap.return_kinds | lmap.raise_kinds,+ exit_macros=_exit_macros(lmap),+ yields=lmap.yield_kinds,+ awaits=lmap.await_kinds,+ await_scopes=lmap.await_scope_kinds,+ lambdas=lmap.lambda_kinds,+ receiver=receiver,+ )+ return kinds+++def _receiver_kinds(receiver: Receiver) -> frozenset[str]:+ """What :func:`_receiver_ref` can answer for: a write, or a receiver leaf."""+ return receiver.write_kinds | receiver.names | {SUPER, "identifier"}+++def function_facts(+ fn_node: Node, lmap: LanguageNodeMap, receiver: Receiver | None, tally: BodyTally+) -> FunctionFacts:+ """:class:`FunctionFacts` from a filled :func:`body_tally`."""+ body = fn_node.child_by_field_name("body") or fn_node+ sink = tally.on_receiver if isinstance(tally.on_receiver, _ReceiverSink) else None+ uses, assigns = _whole_receiver_facts(+ receiver, sink.uses if sink else False, frozenset(sink.assigns) if sink else frozenset()+ )+ return FunctionFacts(+ awaits=tally.awaits,+ is_generator=tally.yields > 0,+ uses_receiver=uses,+ receiver_assigns=assigns,+ early_exits=max(0, tally.exits - _ends_in_exit(body, lmap, tally.kinds)),+ )+++def _ends_in_exit(body: Node, lmap: LanguageNodeMap, kinds: FactKinds) -> bool:+ """Whether a block body's last statement leaves the function, so that exit+ is not an early one. The statement is the exit (Python ``return x``, TS+ ``throw e;``) or wraps it as its first named child (Rust ``return x;`` and+ ``bail!(..);`` are expression statements). An expression body has no+ statements, so its exits are early by definition and none are subtracted."""+ if body.type not in lmap.block_kinds or not body.named_children:+ return False+ tail = body.named_children[-1]+ if is_exit(tail, kinds):+ return True+ return bool(tail.named_children) and is_exit(tail.named_children[0], kinds)+++def _whole_receiver_facts(+ receiver: Receiver | None, uses: bool, assigned: frozenset[str]+) -> tuple[bool | None, tuple[str, ...] | None]:+ if receiver is None:+ return None, None+ if not receiver.names:+ return False, ()+ if receiver.implicit:+ return (True if uses else None), None+ return uses, None if _UNREAD_TARGET in assigned else tuple(sorted(assigned))+++def _free_write_lines(def_use: FunctionDefUse) -> list[int]:+ """Sorted lines writing a name the function neither declares nor takes+ as a parameter: in Java or C++ that is a field (or a C++ global) written+ without ``this``, so the span's receiver writes are not all known."""+ params = {p.name for p in def_use.params}+ declared = {d.var for d in def_use.definitions if d.declares or d.declared_at is not None}+ return sorted(+ d.line for d in def_use.definitions if d.var not in params and d.var not in declared+ )+++def _receiver_facts(+ receiver: Receiver | None, pre: _Prefix, i: int, j: int, free_writes: int+) -> tuple[bool | None, tuple[str, ...] | None]:+ """``uses_receiver`` / ``receiver_assigns`` for the span over statements+ ``i..j`` (see :class:`Extraction`)."""+ if receiver is None:+ return None, None+ if not receiver.names:+ return False, ()+ named = pre.receiver[j + 1] > pre.receiver[i]+ found = frozenset().union(*pre.assigns[i : j + 1])+ fields = None if _UNREAD_TARGET in found else tuple(sorted(found))+ if receiver.implicit:+ return (True if named else None), (None if free_writes else fields)+ return named, fields+++# Index access on a receiver field (``self.cache[k] = v``) assigns into it.+_INDEX_KINDS = frozenset({"subscript", "subscript_expression", "index_expression", "array_access"})+# Unpacking targets whose named children are targets themselves+# (``a, self.b = ...``, ``[this.x, y] = ...``).+_UNPACK_KINDS = frozenset(+ {+ "pattern_list",+ "tuple_pattern",+ "list_pattern",+ "expression_list",+ "tuple",+ "list",+ "array_pattern",+ "object_pattern",+ "parenthesized_expression",+ "list_splat_pattern",+ "rest_pattern",+ }+)+# Marks a target rooted at the receiver whose shape is not read, so the+# span's assigned fields are unknown.+_UNREAD_TARGET = ""+++def _receiver_ref(node: Node, receiver: Receiver, assigns: set[str]) -> bool:+ """True when *node* names the receiver; an assignment to one of its+ fields adds the field's name to *assigns* (the receiver leaf under it is+ visited next)."""+ if node.type in receiver.write_kinds:+ target = (+ node.child_by_field_name("left")+ or node.child_by_field_name("argument")+ or (node.named_children[0] if node.named_children else None)+ )+ if target is not None:+ _collect_assigned(target, receiver, assigns)+ return False+ return _is_receiver(node, receiver)+++def _collect_assigned(target: Node, receiver: Receiver, assigns: set[str]) -> None:+ """The receiver fields one assignment target writes, unpacking included;+ a target of another shape that mentions the receiver makes them unknown."""+ if target.type in _UNPACK_KINDS:+ for each in target.named_children:+ _collect_assigned(each, receiver, assigns)+ return+ if target.type in ("assignment_pattern", "object_assignment_pattern"): # ``[this.a = 1]``+ left = target.child_by_field_name("left")+ if left is not None:+ _collect_assigned(left, receiver, assigns)+ return+ if target.type == "pair_pattern": # ``{key: this.a}``+ value = target.child_by_field_name("value")+ if value is not None:+ _collect_assigned(value, receiver, assigns)+ return+ field = _receiver_field(target, receiver)+ if field:+ assigns.add(field)+ elif target.type not in receiver.access_kinds | _INDEX_KINDS and mentions_receiver(+ target, receiver.names+ ):+ assigns.add(_UNREAD_TARGET)+++def _is_receiver(node: Node, receiver: Receiver) -> bool:+ """A leaf naming the receiver: a ``this`` / ``self`` / ``super`` token,+ or an identifier spelling a named receiver (Python ``self``, Go ``s``)+ or ``super`` (Python's no-argument ``super()``)."""+ if node.children or not node.text:+ return False+ if node.type in receiver.names or node.type == SUPER:+ return True+ if node.type != "identifier":+ return False+ text = node.text.decode("utf-8", "replace")+ return text in receiver.names or text == SUPER+++def _receiver_field(target: Node, receiver: Receiver) -> str | None:+ """``x`` for an assignment target ``self.x``, ``self.x.y`` or ``self.x[k]``."""+ cur: Node | None = target+ while cur is not None and (cur.type in receiver.access_kinds or cur.type in _INDEX_KINDS):+ if cur.type in receiver.access_kinds:+ field = self_member_name(cur, receiver.names)+ if field is not None:+ return field+ cur = member_object(cur)+ return Nonediff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice_state.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice_state.pynew file mode 100644index 000000000..b82602cf3--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice_state.py@@ -0,0 +1,240 @@+"""State a span shares with code it does not hold: closure-shared locals and loop-carried values."""++from __future__ import annotations++from bisect import bisect_left, bisect_right+from collections import defaultdict+from typing import TYPE_CHECKING, NamedTuple++from .slice_liveness import _declaration_lines+from .slice_nodes import _descend, _loop_header_names, _receiver_root++if TYPE_CHECKING:+ from tree_sitter import Node++ from ..complexity.languages import LanguageNodeMap+ from .analyze import FunctionAnalysis+ from .defuse import FunctionDefUse+++class _SharedState(NamedTuple):+ """Per variable a closure shares with the function, sorted line lists:+ the closures' reads and writes (``Captured.shared`` / ``.writes``), the+ function's own defs, all its references, and its binding declarations."""++ reads: dict[str, list[int]]+ writes: dict[str, list[int]]+ defs: dict[str, list[int]]+ refs: dict[str, list[int]]+ binds: dict[str, frozenset[int]]+++def _closure_state(+ def_use: FunctionDefUse,+ def_lines: dict[str, list[int]],+ use_lines: dict[str, list[int]],+) -> _SharedState | None:+ """The closure-shared locals of a function, or None when it has none+ (then the gate costs nothing per span)."""+ reads: dict[str, list[int]] = defaultdict(list)+ writes: dict[str, list[int]] = defaultdict(list)+ for out, occurrences in (+ (reads, def_use.captured.shared),+ (writes, def_use.captured.writes),+ ):+ for occ in occurrences:+ if occ.name in def_lines: # a name this function binds+ out[occ.name].append(occ.line)+ if not reads and not writes:+ return None+ names = set(reads) | set(writes)+ return _SharedState(+ reads={var: sorted(lines) for var, lines in reads.items()},+ writes={var: sorted(lines) for var, lines in writes.items()},+ defs={var: def_lines.get(var, []) for var in names},+ refs={var: sorted((*def_lines.get(var, ()), *use_lines.get(var, ()))) for var in names},+ binds=_declaration_lines(def_use, binding=True),+ )+++def _count_in(lines: list[int], s: int, e: int) -> int:+ return bisect_right(lines, e) - bisect_left(lines, s)+++def _closure_state_crosses(+ st: _SharedState, span: list[Node], s: int, e: int, block: Node, lmap: LanguageNodeMap+) -> bool:+ """True when a local a closure shares crosses the span boundary.++ A closure runs when it is called, not where it is written, so line+ liveness cannot place its reads and writes; an IN/OUT signature copies a+ value where the closure needs the variable itself. Refused:++ - a closure in the span writes a local anything outside the span refers+ to (code, or a closure there): the lifted closure writes the helper's+ copy. A counter declared in the span with its ``inc`` closure is+ refused this way when the count is read after the span;+ - a closure outside the span writes a local the span refers to: the span+ reads or writes a stale copy;+ - the span writes a local that a closure written above it reads: the+ closure runs later and sees the old value.++ The last two are waived when the span's name is a binding of its own+ (:func:`_span_binds`, :func:`_span_only_rebinds`): the closure's variable+ is then another one.+ """+ for var in st.reads.keys() | st.writes.keys():+ writes = st.writes.get(var, [])+ reads = st.reads.get(var, [])+ refs = st.refs[var]+ w_in = _count_in(writes, s, e)+ if w_in and (+ w_in < len(writes)+ or _count_in(reads, s, e) < len(reads)+ or _count_in(refs, s, e) < len(refs)+ ):+ return True+ stale = w_in < len(writes) and _count_in(refs, s, e) > 0+ if stale and not _span_binds(st, var, span, block, lmap):+ return True+ overwritten = bool(reads) and reads[0] < s and _count_in(st.defs[var], s, e) > 0+ if overwritten and not _span_only_rebinds(st, var, s, e, block, lmap):+ return True+ return False+++def _declares_in(node: Node, lines: frozenset[int], lmap: LanguageNodeMap) -> bool:+ """True when *node* is a declaration statement holding one of *lines*."""+ lo, hi = node.start_point[0] + 1, node.end_point[0] + 1+ return node.type in lmap.local_decl_kinds and any(lo <= ln <= hi for ln in lines)+++def _declared_above(binds: frozenset[int], block: Node, s: int, lmap: LanguageNodeMap) -> bool:+ """True when a statement of *block* above the span already declares the+ name: a re-declaration in the same scope (Go's ``v, err := h()``) is an+ assignment to it, not a new binding."""+ return any(+ node.end_point[0] + 1 < s and _declares_in(node, binds, lmap)+ for node in block.named_children+ )+++def _span_binds(+ st: _SharedState, var: str, span: list[Node], block: Node, lmap: LanguageNodeMap+) -> bool:+ """True when one of the span's own top-level statements declares a new+ binding of *var*, so every reference to it in the span is to that one.++ Top-level only: a declaration in a nested block shadows just that block.+ """+ binds = st.binds.get(var, frozenset())+ s = span[0].start_point[0] + 1+ return (+ bool(binds)+ and not _declared_above(binds, block, s, lmap)+ and any(_declares_in(node, binds, lmap) for node in span)+ )+++def _span_only_rebinds(+ st: _SharedState, var: str, s: int, e: int, block: Node, lmap: LanguageNodeMap+) -> bool:+ """True when every write the span makes to *var* creates a new binding+ (a declaration or a loop binder, at any depth), so none of them reaches+ the variable a closure outside the span reads."""+ binds = st.binds.get(var, frozenset())+ defs = st.defs[var][bisect_left(st.defs[var], s) : bisect_right(st.defs[var], e)]+ return all(ln in binds for ln in defs) and not _declared_above(binds, block, s, lmap)+++def _rebinding_defs_dropped(+ analysis: FunctionAnalysis, def_lines: dict[str, list[int]], lmap: LanguageNodeMap+) -> dict[str, list[int]]:+ """*def_lines* without the names exception handlers bind on their head+ line (``except E as e:``, ``catch (err)``). Such a binding never reaches+ the next iteration: Python unbinds it at the handler's end and a catch+ parameter is scoped to its block, so a read of the name elsewhere in the+ loop is another handler's own."""+ heads = _catch_heads(analysis, lmap)+ if not heads:+ return def_lines+ bound: set[tuple[str, int]] = set()+ for head in heads:+ bound |= _head_bindings(head, analysis.def_use, lmap)+ return {var: [ln for ln in lines if (var, ln) not in bound] for var, lines in def_lines.items()}+++def _catch_heads(analysis: FunctionAnalysis, lmap: LanguageNodeMap) -> list[Node]:+ return [+ st.node+ for block in analysis.cfg.blocks+ for st in block.statements+ if st.is_head and st.kind in lmap.catch_kinds and st.node is not None+ ]+++def _head_bindings(+ head: Node, def_use: FunctionDefUse, lmap: LanguageNodeMap+) -> set[tuple[str, int]]:+ # Only the clause's own binder: before its body, so a one-line body's+ # write on the same line still counts.+ body = next((c for c in head.named_children if c.type in lmap.block_kinds), None)+ end = body.start_point if body is not None else head.end_point+ return {+ (d.var, d.line)+ for d in def_use.definitions+ if head.start_point <= (d.line - 1, d.column) < end+ }+++def _loop_carry_free(+ span: list[Node],+ loop: Node,+ s: int,+ e: int,+ def_lines: dict[str, list[int]],+ use_lines: dict[str, list[int]],+ lmap: LanguageNodeMap,+) -> bool:+ """True when a span nested in *loop* carries no state between iterations.++ Refused: a variable the span writes that the loop reads with no in-span+ write before the read (the read takes the previous iteration's value, even+ when it sits inside the span above the write), and a call on a name the+ loop header reads (``entries.remove(entry)`` in ``for entry in entries``).+ """+ if _carries_a_write(loop.start_point[0] + 1, s, e, def_lines, use_lines):+ return False+ carried = _loop_header_names(loop, lmap)+ return not carried or not _mutates_carried(span, carried, lmap)+++def _carries_a_write(+ loop_start: int,+ s: int,+ e: int,+ def_lines: dict[str, list[int]],+ use_lines: dict[str, list[int]],+) -> bool:+ for var, lines in def_lines.items():+ in_span_writes = [ln for ln in lines if s <= ln <= e]+ if not in_span_writes:+ continue+ for read in use_lines.get(var, []):+ if loop_start <= read <= e and not any(write < read for write in in_span_writes):+ return True+ return False+++def _mutates_carried(span: list[Node], carried: set[str], lmap: LanguageNodeMap) -> bool:+ scope_kinds = lmap.function_kinds | lmap.lambda_kinds+ for st in span:+ for call in _descend(st, lmap.call_kinds, scope_kinds):+ callee = call.child_by_field_name("function")+ if callee is None:+ named = call.named_children+ callee = named[0] if named else None+ root = _receiver_root(callee) if callee is not None else None+ if root is not None and root in carried:+ return True+ return Falsediff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice_writes.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice_writes.pynew file mode 100644index 000000000..31abd6d4b--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice_writes.py@@ -0,0 +1,143 @@+"""Structural proof that a span writes each OUT variable on every path through it."""++from __future__ import annotations++from typing import TYPE_CHECKING++from .slice_nodes import _identifiers, _unwrap_container++if TYPE_CHECKING:+ from tree_sitter import Node++ from ..complexity.languages import LanguageNodeMap+++def _outs_definitely_assigned(+ span: list[Node],+ returns: tuple[str, ...],+ def_lines: dict[str, list[int]],+ lmap: LanguageNodeMap,+) -> bool:+ """True when every OUT variable is written on *every* path through the span.++ A conditionally written OUT is the unsound case line-based liveness cannot+ see: the helper returns the unwritten value on the paths that skip the+ write, and the caller's assignment then clobbers the live one. The proof+ needs branch structure, so it runs structurally here and the span is+ refused whenever the proof does not go through.+ """+ return all(_stmts_assign(span, var, def_lines.get(var, []), lmap) for var in returns)+++def _stmts_assign(stmts: list[Node], var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:+ """True when *stmts* writes *var* on every path through them."""+ return any(_stmt_assigns(st, var, defs, lmap) for st in stmts)+++def _stmt_assigns(st: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:+ """True only when *st* is proved to write *var* however it is entered.++ Positive proof only: a statement kind this cannot classify returns False.+ Defaulting to True would let any statement sharing a line with a+ conditional write stand in as the proof.+ """+ lo = st.start_point[0] + 1+ hi = st.end_point[0] + 1+ if not any(lo <= d <= hi for d in defs):+ return False+ if _is_conditional(st, lmap):+ return _conditional_assigns(st, var, defs, lmap)+ if st.type in lmap.block_kinds:+ container = _unwrap_container(st, lmap.block_kinds)+ return _stmts_assign(container.named_children, var, defs, lmap)+ if st.type in lmap.with_kinds or st.type in lmap.statement_wrapper_kinds:+ return _stmts_assign(list(st.named_children), var, defs, lmap)+ return _unconditional_write(st, var, lmap)+++def _unconditional_write(st: Node, var: str, lmap: LanguageNodeMap) -> bool:+ """True when *st* contains a write to *var* no branch or loop guards.++ The walk stops at any control container (a loop body may run zero times, a+ branch arm may not be taken, a catch arm may not fire) and at nested+ scopes, so only writes on the statement's own straight-line path count.+ """+ write_kinds = lmap.assignment_kinds | lmap.augmented_assign_kinds | lmap.local_decl_kinds+ if not write_kinds:+ return False+ barriers = (+ lmap.loop_kinds+ | lmap.try_kinds+ | lmap.catch_kinds+ | lmap.switch_kinds+ | lmap.case_kinds+ | lmap.branch_kinds+ | lmap.if_kinds+ | lmap.function_kinds+ | lmap.lambda_kinds+ )+ if st.type in barriers:+ return False+ stack = [st]+ while stack:+ node = stack.pop()+ if node.id != st.id and node.type in barriers:+ continue+ if node.type in write_kinds and _writes_var(node, var):+ return True+ stack.extend(node.children)+ return False+++def _writes_var(node: Node, var: str) -> bool:+ """True when *var* is on the binding side of this write node."""+ for field in ("left", "pattern", "declarator", "name"):+ target = node.child_by_field_name(field)+ if target is not None:+ return var in _identifiers(target)+ return var in _identifiers(node)+++def _is_conditional(node: Node, lmap: LanguageNodeMap) -> bool:+ return node.type in lmap.if_kinds or (+ node.type in lmap.branch_kinds and node.child_by_field_name("condition") is not None+ )+++def _conditional_assigns(node: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:+ """True when a conditional chain is exhaustive and every arm writes *var*.++ Two chain shapes. Python flattens ``elif``/``else`` into repeated+ ``alternative`` fields on one ``if``, so the terminal else is a sibling of+ the chained arms. The C family nests instead: the single ``alternative`` is+ another ``if``, whose own else ends the chain. Both must be exhaustive.+ """+ consequence = node.child_by_field_name("consequence") or node.child_by_field_name("body")+ if consequence is None:+ return False+ alternatives = list(node.children_by_field_name("alternative"))+ if not alternatives:+ return False # no else at all: some path skips the write+ if not _branch_assigns(consequence, var, defs, lmap):+ return False+ for alt in alternatives:+ if _is_conditional(alt, lmap):+ # Nested chain (C family): it carries the chain's terminal else.+ if len(alternatives) == 1:+ return _conditional_assigns(alt, var, defs, lmap)+ # Flattened chain (Python's ``elif``): the else is a sibling below.+ arm = alt.child_by_field_name("consequence") or alt.child_by_field_name("body")+ if arm is None or not _branch_assigns(arm, var, defs, lmap):+ return False+ elif not _branch_assigns(alt, var, defs, lmap):+ return False+ return any(not _is_conditional(alt, lmap) for alt in alternatives)+++def _branch_assigns(node: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:+ if _is_conditional(node, lmap):+ return _conditional_assigns(node, var, defs, lmap)+ if node.type in lmap.block_kinds:+ container = _unwrap_container(node, lmap.block_kinds)+ return _stmts_assign(container.named_children, var, defs, lmap)+ return _stmts_assign(list(node.named_children), var, defs, lmap)diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.pyindex 669926a32..47375dd8f 100644--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py@@ -1,11 +1,12 @@"""Python ``PerfDialect``.-Extracted verbatim from the original ``_py_sink_kind`` (``io_boundaries.py``)-and the Python branches of the walker (``_blocking_sync_api`` / ``_is_constant_for``-/ the ``_PY_STRING_KINDS`` string-concat predicate). The 6a refactor changes-zero Python behavior — the defect golden + perf suite lock that.+Vocabularies live in :mod:`.python_lexicon`, syntax helpers in+:mod:`.python_nodes`, loop sources in :mod:`.python_sources` and batched forms+in :mod:`.python_batch`. Their names are re-exported here."""+# ruff: noqa: F401+from __future__ import annotationsimport builtins@@ -16,367 +17,88 @@ from .. import loop_lexicon as lexfrom ..loop_facts import BatchForm, LoopSourcefrom .base import BasePerfDialectfrom .iterables import IterableSources--if TYPE_CHECKING:- from tree_sitter import Node-- from ..loop_facts import LoopIterates, LoopMagnitude, SinkProbe-- Source = tuple[LoopIterates, LoopMagnitude | None]--# Network round-trip verbs (shared across languages that name HTTP methods).-HTTP_VERBS: frozenset[str] = frozenset(- {"get", "post", "put", "delete", "patch", "head", "options", "request", "send", "stream"}-)--# Python DBAPI / SQLAlchemy execution sinks, split into three strata so the-# ambiguous accessors can be gated harder than the unambiguous ones (see-# ``sink_kind``).-PY_DB_UNAMBIGUOUS: frozenset[str] = frozenset({"fetchone", "fetchall", "fetchmany"})-# DB verbs other clients spell too: GitPython's ``repo.commit()``, an HTTP or-# GraphQL SDK's ``request.execute()``, a builder's ``.scalar()``.-PY_DB_NEEDS_EVIDENCE: frozenset[str] = frozenset(- {"commit", "execute", "executemany", "scalars", "scalar", "scalar_one", "scalar_one_or_none"}+from .python_batch import PythonBatchForms+from .python_lexicon import (+ _PY_AUG_ASSIGN_KINDS,+ _PY_BINDING_PARENTS,+ _PY_BINDING_WRAPPERS,+ _PY_BUILTINS,+ _PY_CONSTANT_RHS,+ _PY_DEF_KINDS,+ _PY_KEY_READS,+ _PY_LIST_BUILTINS,+ _PY_LIST_RHS_KINDS,+ _PY_LISTINGS,+ _PY_LOCK_METHODS,+ _PY_MODEL_NAME_RE,+ _PY_NONLIST_BUILTINS,+ _PY_NONLIST_RHS_KINDS,+ _PY_ONE_ROW,+ _PY_PASS_THROUGH,+ _PY_PER_KEY_LIMITS,+ _PY_QUERY_CHAIN_METHODS,+ _PY_READ_CAPS,+ _PY_RECURSIVE_LISTINGS,+ _PY_RESOURCE_CTORS,+ _PY_RESOURCE_METHODS,+ _PY_RESULT_PROJECTIONS,+ _PY_SCOPES,+ _PY_SEM_NAME_RE,+ _PY_SQL_COMMENT_RE,+ _PY_SQL_LIMIT_RE,+ _PY_SQL_LITERAL_RE,+ _PY_STATEMENT_BUILDERS,+ _PY_STREAMS,+ _PY_STRING_KINDS,+ _PY_WRITES,+ HTTP_VERBS,+ PY_DB_AMBIGUOUS,+ PY_DB_NEEDS_EVIDENCE,+ PY_DB_UNAMBIGUOUS,+ PY_FS_METHODS,+ PY_OS_FS_METHODS,+ PY_SHUTIL_FS_METHODS,+ PY_SUBPROC_METHODS,+ PY_UNBOUNDED_READ_BOUND_METHODS,)-# Without a db import, the call itself can still show it runs a query: a-# PostgREST chain (``client.table("t")...execute()``) or a statement argument-# (``session.execute(select(...))``, ``cur.execute("SELECT ...")``).-_PY_QUERY_CHAIN_METHODS: frozenset[str] = frozenset({"table", "from_"})-_PY_STATEMENT_BUILDERS: frozenset[str] = frozenset({"select", "insert", "update", "delete", "text"})-_PY_SQL_LITERAL_RE = re.compile(- r"^[rRuUbBfF]*[\"']+\s*(?:SELECT|INSERT|UPDATE|DELETE|WITH|CREATE|DROP|ALTER|PRAGMA)\b",- re.IGNORECASE,-)-PY_DB_AMBIGUOUS: frozenset[str] = frozenset({"all", "first", "one", "one_or_none"})-PY_SUBPROC_METHODS: frozenset[str] = frozenset(- {"run", "call", "check_call", "check_output", "Popen"}-)-# Any of these anywhere in a read's call chain proves it is bounded, gating-# ``unbounded_read_reduced_in_memory`` (analysis.health.perf.unbounded_reduction).-PY_UNBOUNDED_READ_BOUND_METHODS: frozenset[str] = frozenset(- {"limit", "range", "single", "maybe_single", "first", "count", "head", "aggregate"}+from .python_nodes import (+ _PY_SCALARS,+ _bindings,+ _binds,+ _field_text,+ _fixed_set,+ _in_filter,+ _literal_range,+ _module_int_constant,+ _scalar_literal,+ _string_value,+ _text,+ _value_refs,)+from .python_sources import PythonLoopSources-# Filesystem round-trips, in two strata for the same reason the DB verbs are-# stratified: how much evidence the name alone carries.-#-# ``PY_FS_METHODS`` are matched on the method name alone, because in this-# ecosystem only a ``pathlib.Path`` spells them — the same reasoning the TS-# dialect applies to ``readFileSync``. That matters because the dominant real-# shape binds the path to a local first (``src_path.read_text()``), so the-# receiver is an ordinary variable and ``io_names`` cannot classify it; keying-# on the module would miss nearly every genuine site. The Kotlin dialect-# deliberately excludes its ``readText`` / ``readBytes`` homonyms because-# kotlinx-io and Ktor mirror those names on in-memory buffers; Python has no-# such homonym (``io.StringIO`` and every stream type spell it ``.read()``),-# which is what makes the bare-name match safe here and not there.-#-# Metadata syscalls (``exists`` / ``is_file`` / ``is_dir`` / ``stat`` /-# ``glob``) are deliberately absent. A stat in a loop is cheap and usually-# correct, so flagging it is the obvious false-positive class — the same call-# the C++ dialect makes when it excludes buffered stdio.-PY_FS_METHODS: frozenset[str] = frozenset(- {"read_text", "read_bytes", "write_text", "write_bytes"}-)-# ``os`` and ``shutil`` verbs, gated on the literal module root the way-# ``subprocess`` is. ``os`` is far too broad to classify wholesale (``os.environ``-# / ``os.getpid`` / ``os.path.join`` are not I/O), so it never goes through-# ``io_names``; the root name plus an explicit verb set is the whole gate.-#-# CEILING: ``callee_root_name`` resolves the LEFTMOST identifier of the whole-# callee chain, so ``os.path.join(a, b).verb()`` also arrives here as-# ``root == "os"``. The verb set must therefore hold no name that a ``str`` /-# ``list`` / ``dict`` also answers to. ``replace`` was the one that did, and it-# was a live false positive on the canonical Windows path idiom-# (``os.path.join(root, name).replace("\\", "/")`` — five instances across the-# OSS corpus, each yielding a bogus ``io_in_loop`` AND a ``nested_loop_with_io``).-# It is omitted at near-zero recall cost: ``os.replace`` is atomic ``os.rename``,-# which is already listed. Upgrade path if this set ever needs a colliding verb:-# thread the callee's dotted segment count into ``sink_kind`` and require-# exactly two, which is the real invariant being approximated here.-PY_OS_FS_METHODS: frozenset[str] = frozenset(- {- "walk",- "listdir",- "scandir",- "remove",- "unlink",- "rename",- "makedirs",- "mkdir",- "rmdir",- "removedirs",- }-)-PY_SHUTIL_FS_METHODS: frozenset[str] = frozenset(- {- "copy",- "copy2",- "copyfile",- "copytree",- "move",- "rmtree",- "make_archive",- "unpack_archive",- }-)--# Python string-literal node kinds (f-strings parse as ``string`` too).-_PY_STRING_KINDS: frozenset[str] = frozenset({"string", "concatenated_string"})-_PY_AUG_ASSIGN_KINDS: frozenset[str] = frozenset({"augmented_assignment"})--# Heavy-resource constructors: building one of these per loop iteration opens a-# fresh connection / client / pool instead of reusing a hoisted one. Keyed as-# ``(root, method)`` pairs whose root is a distinctive I/O library (so the match-# is unambiguous without import resolution) ...-_PY_RESOURCE_CTORS: frozenset[tuple[str, str]] = frozenset(- {- ("sqlite3", "connect"),- ("psycopg", "connect"),- ("psycopg2", "connect"),- ("pymysql", "connect"),- ("MySQLdb", "connect"),- ("aiosqlite", "connect"),- ("httpx", "Client"),- ("httpx", "AsyncClient"),- ("requests", "Session"),- ("aiohttp", "ClientSession"),- ("boto3", "client"),- ("boto3", "resource"),- ("redis", "Redis"),- ("redis", "StrictRedis"),- ("pymongo", "MongoClient"),- }-)-# ... plus a few constructors distinctive enough on their own (the rightmost-# name), so an aliased ``from sqlalchemy import create_engine`` still resolves.-_PY_RESOURCE_METHODS: frozenset[str] = frozenset({"create_engine", "MongoClient"})+if TYPE_CHECKING:+ from tree_sitter import Node-# Lock acquisition (the contention side only — never ``release``, which would-# double-count the same critical section). ``.acquire()`` is the threading /-# asyncio / multiprocessing / filelock primitive verb.-_PY_LOCK_METHODS: frozenset[str] = frozenset({"acquire"})-# RHS node kinds that prove a name is bound to a list (the membership gate).-_PY_LIST_RHS_KINDS: frozenset[str] = frozenset({"list", "list_comprehension"})-# Builtins whose call result is provably a list.-_PY_LIST_BUILTINS: frozenset[str] = frozenset({"list", "sorted"})-# RHS shapes that make a name a NON-list container (a set/dict literal or-# comprehension, or a ``set()`` / ``dict()`` / ``frozenset()`` call). A name-# bound to one of these ANYWHERE in the file is ambiguous and excluded from the-# membership gate, even if another scope binds the same name to a list — the-# ``seen = []`` in one function / ``seen: set = set()`` in another collision.-_PY_NONLIST_RHS_KINDS: frozenset[str] = frozenset(- {"set", "dictionary", "set_comprehension", "dictionary_comprehension"}-)-_PY_NONLIST_BUILTINS: frozenset[str] = frozenset({"set", "dict", "frozenset"})--# Promotion facts (perf/loop_facts.py). Calls and attributes that read a query-# result out without changing how many rows it holds.-_PY_RESULT_PROJECTIONS: frozenset[str] = frozenset(- {"all", "scalars", "fetchall", "json", "data", "rows"}-)-# Raw SQL capped in its own text: ``sa.text("... LIMIT 1000")``.-_PY_SQL_LIMIT_RE = re.compile(rb"(?is)\bselect\b.*\blimit\s+\d+\b")-_PY_SQL_COMMENT_RE = re.compile(rb"--[^\n]*|/\*.*?\*/", re.S)-# A per-key call that limits or orders its rows is not the read one IN query makes.-_PY_PER_KEY_LIMITS: frozenset[str] = frozenset(- {- "limit", "single", "maybe_single", "first", "one", "scalar_one",- "scalar_one_or_none", "order", "order_by", "range", "offset",- }-)-_PY_ONE_ROW: frozenset[str] = _PY_PER_KEY_LIMITS | {"scalar"}-_PY_READ_CAPS: frozenset[str] = _PY_ONE_ROW - {"order", "order_by", "offset"}-_PY_WRITES: frozenset[str] = frozenset({"update", "insert", "upsert", "values"})-# Keys a query yields (``.in_(select(...))``) are as many as that table's rows.-_PY_KEY_READS: frozenset[str] = _PY_STATEMENT_BUILDERS | {- "subquery", "scalar_subquery", "query", "cte", "alias", "union", "union_all",- "execute", "table", "from_",+_PY_DB_METHODS: frozenset[str] = PY_DB_UNAMBIGUOUS | PY_DB_NEEDS_EVIDENCE | PY_DB_AMBIGUOUS+_PY_ROOTED_PROCESS_VERBS: dict[str, frozenset[str]] = {"subprocess": PY_SUBPROC_METHODS}+_PY_ROOTED_FILE_VERBS: dict[str, frozenset[str]] = {+ "open": frozenset({"open"}),+ "os": PY_OS_FS_METHODS,+ "shutil": PY_SHUTIL_FS_METHODS,}-_PY_SCOPES: frozenset[str] = frozenset({"function_definition", "lambda", "class_definition"})-_PY_BUILTINS: frozenset[str] = frozenset(dir(builtins))-_PY_SEM_NAME_RE = re.compile(r"(?i)(sem|semaphore|limiter|limit)$")-# A model class is CapWords; an ALL_CAPS constant (``API_URL``) is not.-_PY_MODEL_NAME_RE = re.compile(r"^[A-Z]\w*[a-z]\w*$")-# Directory listings; the recursive ones walk a whole tree.-_PY_LISTINGS: frozenset[str] = frozenset(- {- "glob", "iglob", "rglob", "iterdir", "walk", "listdir", "scandir", "traverse",- "iter_glob", "glob_via", "walk_repo", "_rglob",- }-)-_PY_RECURSIVE_LISTINGS: frozenset[str] = frozenset(- {"rglob", "walk", "traverse", "walk_repo", "_rglob"}-)-# Reads that yield lines or chunks as they arrive.-_PY_STREAMS: frozenset[str] = frozenset(- {- "readlines", "iter_lines", "iter_content", "iter_bytes", "iter_text",- "aiter_lines", "aiter_bytes", "aiter_text", "aiter_raw",- }-)-# Wrappers and chunkers whose first argument is what the loop walks.-_PY_PASS_THROUGH: frozenset[str] = frozenset(- {- "enumerate", "sorted", "reversed", "list", "set", "tuple", "iter", "zip",- "batched", "chunked", "ichunked", "chunks", "iter_chunks", "grouper",- }-)-_PY_CONSTANT_RHS: frozenset[str] = frozenset({"list", "tuple", "set", "dictionary"})---_PY_DEF_KINDS = frozenset({"function_definition", "lambda", "class_definition"})-# Nodes an identifier sits in when it is the name being bound, not read.-_PY_BINDING_PARENTS = frozenset(- {- "global_statement", "nonlocal_statement", "as_pattern_target", "aliased_import",- "import_from_statement", "import_statement", "parameters", "lambda_parameters",- "named_expression",- }-)-_PY_BINDING_WRAPPERS = frozenset(- {- "pattern_list", "tuple_pattern", "list_pattern", "typed_parameter", "default_parameter",- "typed_default_parameter", "list_splat_pattern", "dotted_name",- }-)---def _binds(ident: Node) -> bool:- """Whether *ident* is a name being bound (assigned, imported, a parameter)."""- cur, parent = ident, ident.parent- while parent is not None and parent.type in _PY_BINDING_WRAPPERS:- cur, parent = parent, parent.parent- if parent is None:- return False- if parent.type in ("assignment", "augmented_assignment", "for_statement", "for_in_clause"):- return parent.child_by_field_name("left") == cur- if parent.type in ("function_definition", "class_definition"):- return parent.child_by_field_name("name") == cur- return parent.type in _PY_BINDING_PARENTS---def _bindings(scope: Node, name: bytes) -> list[Node]:- """Identifiers binding *name* in *scope* itself, not in a nested scope."""- found: list[Node] = []- stack = list(scope.children)- while stack:- node = stack.pop()- if node.type in _PY_DEF_KINDS:- stack.extend(c for c in (node.child_by_field_name("name"),) if c is not None)- continue- if node.type == "identifier" and node.text == name and _binds(node):- found.append(node)- stack.extend(node.children)- return found-def _module_int_constant(name: Node) -> bool:- """An ALL_CAPS name its own module binds exactly once, to an integer literal- (``MAX = 5``), and no enclosing function rebinds or shadows.-- The spelling alone proves nothing (``NUM_USERS`` grows), so a name rebound,- imported, augmented, written through ``global``, shadowed by a parameter or- a local, or bound to anything but a literal stays unknown.- """- text = name.text or b""- if name.type != "identifier" or len(text) < 2 or not text.decode().isupper():- return False- scope = name.parent- while scope is not None and scope.parent is not None:- if scope.type in ("function_definition", "lambda") and _bindings(scope, text):- return False- scope = scope.parent- if scope is None or any(- n.type == "global_statement" and any(c.text == text for c in n.named_children)- for n in BasePerfDialect._walk(scope)- ):- return False- bound = _bindings(scope, text)- if len(bound) != 1 or bound[0].parent is None or bound[0].parent.type != "assignment":- return False- right = bound[0].parent.child_by_field_name("right")- return right is not None and right.type == "integer"---def _text(node: Node) -> str:- return (node.text or b"").decode("utf-8", "replace")---def _field_text(node: Node, field: str) -> str:- child = node.child_by_field_name(field)- return _text(child) if child is not None else ""---def _value_refs(expr: Node) -> list[Node]:- """The names and dotted paths *expr* reads values from: not a callee- (``tuple`` in ``tuple(xs)``), a keyword, or an ALL_CAPS constant."""- refs = []- stack = [expr]- while stack:- node = stack.pop()- if node.type not in ("identifier", "attribute"):- stack.extend(node.children)- continue- parent = node.parent- callee = parent is not None and parent.child_by_field_name("function") == node- keyword = parent is not None and parent.type == "keyword_argument" and (- parent.child_by_field_name("name") == node- )- last = node.child_by_field_name("attribute") if node.type == "attribute" else node- constant = last is not None and _text(last).strip("_").isupper()- if callee and node.type == "attribute":- # ``self.ids.copy()`` still reads ``self.ids``.- stack.extend(filter(None, [node.child_by_field_name("object")]))- elif not (callee or keyword or constant):- refs.append(node)- return refs---def _string_value(node: Node) -> str | None:- return _text(node).strip("\"'") if node.type == "string" else None---def _in_filter(call: Node, args: list[Node]) -> tuple[str, Node] | None:- """``(column, keys)`` of ``Model.col.in_(keys)`` or PostgREST ``.in_("col", keys)``."""- if len(args) == 2:- column = _string_value(args[0])- return (column, args[1]) if column else None- fn = call.child_by_field_name("function")- receiver = fn.child_by_field_name("object") if fn is not None else None- if len(args) != 1 or receiver is None or receiver.type != "attribute":+def _db_sink(method: str, is_attribute: bool, evidence: bool) -> str | None:+ """A DB sink is a member call on a session or cursor (never the builtin+ ``all(...)``); verbs other clients also spell need db evidence."""+ if not is_attribute:return None- column = _field_text(receiver, "attribute")- return (column, args[0]) if column else None---def _fixed_set(value: Node | None) -> bool:- """A non-empty literal of scalars; an empty one is a container filled later."""- return (- value is not None- and value.type in _PY_CONSTANT_RHS- and bool(value.named_children)- and all(_scalar_literal(item) for item in value.named_children)- )---_PY_SCALARS = frozenset({"string", "concatenated_string", "integer", "float", "true", "false", "none"})+ return "db" if method in PY_DB_UNAMBIGUOUS or evidence else None-def _scalar_literal(item: Node) -> bool:- """``"a"`` / ``3`` / ``("a", 1)`` / ``"k": v`` by its key: an entry the source- spells out. A name or call in the literal (``[ROOT / "a", repo.path]``) may- stand for anything, so the table is not shown to be a fixed set."""- if item.type == "pair":- key = item.child_by_field_name("key")- return key is not None and key.type in _PY_SCALARS- if item.type == "tuple":- return all(child.type in _PY_SCALARS for child in item.named_children)- return item.type in _PY_SCALARS---class PythonPerfDialect(IterableSources):+class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):language = "python"lock_acquire_functions = frozenset({"acquire", "__enter__"})spin_loop_header = re.compile(r"while\s+True\s*:")@@ -403,14 +125,8 @@ class PythonPerfDialect(IterableSources):string_literal_kinds = _PY_STRING_KINDSaug_assign_kinds = _PY_AUG_ASSIGN_KINDS- # Everything whose cost is bounded by ONE inode: a single file read/write, a- # single unlink/mkdir/rename, one directory listing. Outside a loop that is- # ordinary work, not a latency risk, so it must not reach ``hot_path_sync_io``- # (see ``BasePerfDialect.hot_path_excluded_methods``). What survives is the- # unbounded set — a whole-tree walk, a recursive copy/delete, an archive- # round-trip — plus the bare ``open(...)`` builtin the marker was originally- # calibrated on. Keyed on the method name alone, so a same-named non-``os``- # receiver is excluded too: recall-only, never a false positive.+ # One-inode operations are ordinary work outside a loop; only unbounded ones+ # (tree walks, recursive copy/delete, archives) and bare ``open`` stay hot-path I/O.hot_path_excluded_methods = (PY_FS_METHODS| (PY_OS_FS_METHODS - {"walk"})@@ -427,53 +143,22 @@ class PythonPerfDialect(IterableSources):io_names: dict[str, str],has_db_import: bool,) -> str | None:- root_kind = io_names.get(root)- db_evidence = has_db_import or root_kind == "db"-- # ``asyncio.sleep`` / ``time.sleep`` / ``trio.sleep`` is a cooperative- # yield, never an I/O round-trip — but ``await asyncio.sleep(...)`` would- # otherwise hit the awaited-network arm below when ``asyncio`` is- # import-classified as network, FP-ing ``io_in_loop`` / ``serial_await``- # on every backoff/poll loop, measured across a Python corpus. No legitimate- # execution sink is named ``sleep``, so excluding it costs no recall.+ # ``sleep`` is a cooperative yield, never a round-trip, even when its+ # module is import-classified as network.if method == "sleep":return None-- if method in PY_DB_UNAMBIGUOUS:- # A real DB sink is always a method on a session/cursor/result- # object; a bare identifier call (the builtin ``all(...)``) is not.- return "db" if is_attribute else None- if method in PY_DB_NEEDS_EVIDENCE:- # A DB verb only with db evidence in the file / on the receiver.- return "db" if (is_attribute and db_evidence) else None- if method in PY_DB_AMBIGUOUS:- # ``.all`` / ``.first`` / ``.one`` collide with ordinary collection- # helpers — the riskiest stratum. Gate on db evidence.- return "db" if (is_attribute and db_evidence) else None- if root == "subprocess" and method in PY_SUBPROC_METHODS:- return "subprocess"- if method == "Popen":+ root_kind = io_names.get(root)+ if method in _PY_DB_METHODS:+ return _db_sink(method, is_attribute, has_db_import or root_kind == "db")+ if method == "Popen" or method in _PY_ROOTED_PROCESS_VERBS.get(root, ()):return "subprocess"- if root == "open" and method == "open": # bare ``open(...)`` builtin- return "filesystem"- if is_attribute and method in PY_FS_METHODS:- # ``p.read_text()`` / ``Path(x).write_bytes(...)``. ``is_attribute``- # mirrors the DB branches: pathlib always spells these as a member- # call, so requiring it costs no recall and drops a bare local- # helper that happens to share the name.- return "filesystem"- if root == "os" and method in PY_OS_FS_METHODS:- return "filesystem"- if root == "shutil" and method in PY_SHUTIL_FS_METHODS:+ # A verb under its literal module root (bare ``open`` too), or a pathlib member call.+ if method in _PY_ROOTED_FILE_VERBS.get(root, ()) or (is_attribute and method in PY_FS_METHODS):return "filesystem"- if method == "urlopen":- return "network"- if is_attribute and root_kind == "network" and method in HTTP_VERBS:- # ``requests.get(...)`` is a member call on an imported client; a bare- # ``get(...)`` (e.g. a builtin/local) never has the verb-call shape, so- # gate on ``is_attribute`` like the DB branches above.- return "network"- if awaited and root_kind == "network":+ # An HTTP verb only as a member call on an imported client: a bare ``get(...)`` is not one.+ if method == "urlopen" or (+ root_kind == "network" and (awaited or (is_attribute and method in HTTP_VERBS))+ ):return "network"return None@@ -561,16 +246,22 @@ class PythonPerfDialect(IterableSources):return Falseloop = loop.parentbody = loop.child_by_field_name("body") if loop is not None else None+ if body is None:+ return Falsesession = (self.callee_root_name(call) or "").encode()- for node in self._walk(body, prune=_PY_SCOPES) if body is not None else ():- args = node.child_by_field_name("arguments") if node.type == "call" else None- if args is None or node.end_byte > call.start_byte:- continue- texts = {arg.text for arg in args.children}- same_session = session in texts or self.callee_root_name(node) == session.decode()- if key.text in texts and same_session:- return True- return False+ return any(+ self._passes_key(node, call, key, session)+ for node in self._walk(body, prune=_PY_SCOPES)+ )++ def _passes_key(self, node: Node, call: Node, key: Node, session: bytes) -> bool:+ """*node* is a call before *call* passing *key* on the same *session*."""+ args = node.child_by_field_name("arguments") if node.type == "call" else None+ if args is None or node.end_byte > call.start_byte:+ return False+ texts = {arg.text for arg in args.children}+ same_session = session in texts or self.callee_root_name(node) == session.decode()+ return key.text in texts and same_sessiondef _orm_get(self, call: Node) -> tuple[str, Node] | None:"""``(Model, key)`` of a ``x.get(Model, key)`` primary-key lookup."""@@ -587,12 +278,8 @@ class PythonPerfDialect(IterableSources):return model, positional[1]def is_constant_loop(self, node: Node) -> bool:- """True if a Python for-loop iterates a compile-time-constant bound.-- Catches ``for _ in range(<int literals>)``, ``for x in (<literal>)``,- and ``for x in ALL_CAPS`` (a named module constant by convention).- ``while`` loops are never constant.- """+ """``for _ in range(<int literals>)``, ``for x in (<literal>)`` or+ ``for x in ALL_CAPS`` (a module constant by convention)."""if node.type != "for_statement":return Falseright = node.child_by_field_name("right")@@ -600,28 +287,11 @@ class PythonPerfDialect(IterableSources):return Falseif right.type in ("list", "tuple", "set"):return True- # A bare ALL_CAPS identifier is a named constant by convention.if right.type == "identifier" and right.text is not None:name = right.text.decode("utf-8", "replace")if name.isupper() and len(name) > 1:return True- if right.type == "call":- fn = right.child_by_field_name("function")- if fn is None or (fn.text or b"").decode("utf-8", "replace") != "range":- return False- args = right.child_by_field_name("arguments")- if args is None:- return False- for a in args.children:- if not a.is_named:- continue- if a.type == "integer":- continue- if a.type == "unary_operator" and any(c.type == "integer" for c in a.children):- continue- return False # a non-literal arg (e.g. len(x)) ⇒ data-dependent- return True- return False+ return right.type == "call" and _literal_range(right)# Builtins that wrap a collection without changing what is iterated, so the# same-collection O(n^2) shape sees through them: ``for x in enumerate(items)``@@ -693,16 +363,8 @@ class PythonPerfDialect(IterableSources):return len(named) == 3 and named[2].text not in (b"1", b"-1")def is_string_concat(self, node: Node) -> bool:- """``s += "x"`` accumulation — but skip an accumulator that is *reset*- each iteration of an enclosing loop.-- ``buf = base[:N]; ... buf += part`` inside a loop builds a fresh, bounded- string per iteration (not the O(n^2) cross-iteration accumulation the- marker targets), so it is a false positive. Measured on a Python corpus:- the reset-per-iteration shape was the dominant FP class (Py 77.8%). When- the same name is plainly re-assigned inside an enclosing loop body, the- ``+=`` cannot accumulate across iterations, so do not flag it.- """+ """``s += "x"`` accumulation, unless an enclosing loop body re-assigns the+ name: then the string is rebuilt per iteration, not accumulated across them."""if not super().is_string_concat(node):return Falseleft = node.child_by_field_name("left")@@ -766,21 +428,11 @@ class PythonPerfDialect(IterableSources):) -> str | None:if (root, method) in _PY_RESOURCE_CTORS or method in _PY_RESOURCE_METHODS:return "resource_construction_in_loop"- # ``lock.acquire()`` — a method call on a receiver (never the builtin- # ``acquire(...)``, which does not exist; the attribute gate is a cheap- # extra guard against a bare-name collision).+ # ``lock.acquire()``: the attribute gate guards against a bare-name collision.if method in _PY_LOCK_METHODS and self.callee_is_attribute(node):return "lock_in_loop"- # ``lst.insert(0, x)`` each iteration shifts the whole list -> O(n^2);- # ``collections.deque.appendleft`` / build-then-reverse is O(n). Gated to- # a literal ``0`` first arg (``insert(i, x)`` at a variable index is not- # the front-insertion anti-pattern) AND to a list that is not re-created- # each iteration (a fresh ``buf = [...]; buf.insert(0, x)`` is bounded,- # not O(n^2) — the same reset-per-iteration FP class as string_concat;- # measured on a Python corpus).- # ``collections.deque.insert(0, x)`` is O(1) front-insertion by design (it- # is the deque's own ``appendleft`` machinery), so exclude a receiver- # provably constructed via ``deque()`` / ``collections.deque(...)``.+ # ``lst.insert(0, x)`` shifts the whole list each pass: a literal ``0``, on a+ # list not re-created per iteration and not a ``deque`` (O(1) at the front).if (method == "insert"and self.callee_is_attribute(node)@@ -796,18 +448,10 @@ class PythonPerfDialect(IterableSources):return Nonedef loop_iterable_call_marker(self, node: Node) -> str | None:- """``for _, row in df.iterrows():`` — row-by-row DataFrame iteration.+ """``for _, row in df.iterrows():``, row-by-row DataFrame iteration in the loop header.- ``DataFrame.iterrows()`` boxes every row into a fresh Series, an order of- magnitude slower than a vectorized operation (and slower than- ``itertuples`` / ``to_dict`` when row access is unavoidable). The call- sits in the loop header, so the body ``loop_call_marker`` misses it.- Gated to the distinctive method name on a member-access receiver- (``x.iterrows()``, never a bare ``iterrows(...)``) AND to a file that- imports pandas (same style as the ``pd``/``pandas`` root gate in- ``pd_concat``). Residual limitation: this is still a name match — a- user-defined class with an ``iterrows`` method in a file that also- imports pandas would still be flagged.+ A member call in a file that imports pandas. Ceiling: still a name match,+ so a user class with an ``iterrows`` method in such a file is flagged too."""if node.type != "for_statement":return None@@ -927,16 +571,9 @@ class PythonPerfDialect(IterableSources):return "membership_test_against_list_in_loop" if name in list_names else Nonedef list_bound_names(self, root: Node) -> frozenset[str]:- """Names assigned a provable list anywhere in the file, minus any name- also bound to a non-list container.-- Covers ``name = [...]`` / ``name = [x for x in ...]`` / ``name =- list(...)`` / ``name = sorted(...)``. Conservative on purpose: an opaque- ``name = build()`` is not counted, and a name bound to a set/dict in any- scope of the file is dropped (the ``seen``-as-list-here /- ``seen``-as-set-there collision), so the membership marker only fires- against a name we can prove is always a list.- """+ """Names assigned a provable list anywhere in the file (``[...]``, a list+ comprehension, ``list(...)``, ``sorted(...)``), minus any name also bound to+ a set or dict, or used as a parameter, in any scope."""list_names: set[str] = set()exclude: set[str] = set()stack: list[Node] = [root]@@ -957,10 +594,6 @@ class PythonPerfDialect(IterableSources):elif self._rhs_is_nonlist_container(right):exclude.add(name)elif n.type == "parameters":- # A name also used as a function parameter has no proven binding- # inside that scope (the caller could pass a set), so it collides- # with an unrelated module-level list of the same name. Exclude it,- # mirroring the set/dict-literal exclusion above.exclude.update(self._param_names(n))for c in n.children:stack.append(c)@@ -1054,438 +687,5 @@ class PythonPerfDialect(IterableSources):ok = target is not None and target.type == "identifier" and target.text and value is not Nonereturn [(target.text, value)] if ok else []- def loop_source(self, loop: Node, probe: SinkProbe) -> LoopSource | None:- if loop.type != "for_statement":- return None- right = loop.child_by_field_name("right")- baseline = self._magnitude(right, loop, probe, hop=True) or "unknown"- iterates, claim = self._source(right, loop, probe, hop=True)- if iterates == "unknown" and baseline == "unknown":- return None- return LoopSource(iterates, claim or baseline)-- def _source(self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:- """What *expr* walks, and the magnitude that settles when it differs from- what the iterable's syntax alone proves (``None``: keep that).-- *hop* allows one step through the name's last assignment before the loop.- """- if expr is None:- return "unknown", None- if expr.type in ("await", "parenthesized_expression"):- inner = next((c for c in expr.children if c.is_named), None)- return self._source(inner, loop, probe, hop=hop)- if expr.type == "boolean_operator" and any(c.type == "or" for c in expr.children):- return self._source(expr.child_by_field_name("left"), loop, probe, hop=hop)- if expr.type in ("identifier", "attribute") and lex.is_config_name(_text(expr)):- # A registry grows with what users register: the name alone bounds nothing.- return "config_table", None- if expr.type == "identifier":- return self._name_source(expr, loop, probe, hop=hop)- if expr.type == "attribute":- return self._attribute_source(expr, loop, probe, hop=hop)- if expr.type == "subscript" and self._slice_magnitude(expr) == "bounded":- # A fixed-width slice bounds the trip count, not what is walked.- walked = self._source(expr.child_by_field_name("value"), loop, probe, hop=hop)- return walked[0], "bounded"- if expr.type == "call":- return self._call_source(expr, loop, probe, hop=hop)- return "unknown", None-- def _attribute_source(self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:- """``result.rows`` walks ``result``; ``self._TABLES`` bound to a literal is a constant."""- attr = expr.child_by_field_name("attribute")- name = (attr.text if attr is not None else b"") or b""- if name.decode("utf-8", "replace") in _PY_RESULT_PROJECTIONS:- return self._source(expr.child_by_field_name("object"), loop, probe, hop=hop)- if self._literal_constant(expr, name):- return "constant", "bounded"- return "unknown", None-- def _name_source(self, name: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:- rhs = self.reaching_value(loop, name.text or b"") if hop else None- if rhs is not None:- return self._source(rhs, loop, probe, hop=False)- fn = loop.parent- while fn is not None and fn.type != "function_definition":- fn = fn.parent- params = fn.child_by_field_name("parameters") if fn is not None else None- if params is not None and _text(name) in self._param_names(params):- return "parameter", None- return "unknown", None-- def _call_source(self, call: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:- method, fn, receiver, args = self._call_bits(call)- if lex.is_config_name(self._dotted_path(fn) or "", callee=True):- return "config_table", None- if method == "range" and receiver is None:- return self._range_source(call, args, loop, probe, hop=hop)- walked = self._walked_arg_or_receiver(method, receiver, args)- if walked is not None:- return self._source(walked, loop, probe, hop=hop)- if method in _PY_LISTINGS or method in _PY_STREAMS or (method == "open" and not receiver):- return self._file_source(call, method, loop)- if probe(call) in ("db", "network"):- return self._query(call, loop)- if receiver is not None and method in _PY_RESULT_PROJECTIONS:- return self._source(receiver, loop, probe, hop=hop)- return "unknown", None-- def _call_bits(self, call: Node) -> tuple[str, Node | None, Node | None, list[Node]]:- """``(method, callee, receiver, positional args)`` of a call."""- fn = call.child_by_field_name("function")- receiver = fn.child_by_field_name("object") if fn is not None and fn.type == "attribute" else None- arg_list = call.child_by_field_name("arguments")- args = [c for c in arg_list.children if c.is_named] if arg_list is not None else []- return self.callee_method_name(call) or "", fn, receiver, args-- @staticmethod- def _walked_arg_or_receiver(- method: str, receiver: Node | None, args: list[Node]- ) -> Node | None:- """``sorted(xs)`` / ``enumerate(xs)`` walk ``xs``; ``d.items()`` walks ``d``."""- if method in _PY_PASS_THROUGH and args and args[0].type != "keyword_argument":- return args[0]- if receiver is not None and method in ("items", "values", "keys"):- return receiver- return None-- def _file_source(self, call: Node, method: str, loop: Node) -> Source:- """A directory listing (grows when it walks the repository tree) or a stream."""- if method in _PY_LISTINGS:- return "listing", "grows_with_data" if self._walks_repo(call, method, loop) else None- return "stream", None-- def _range_source(- self, call: Node, args: list[Node], loop: Node, probe: SinkProbe, *, hop: bool- ) -> Source:- """``range(3)`` is a constant; ``range(0, len(ids), N)`` walks ``ids``."""- if self._range_magnitude(call, loop, probe, hop=hop) == "bounded":- return "constant", "bounded"- stop = args[1] if len(args) > 1 else (args[0] if args else None)- if stop is not None and stop.type == "call" and self.callee_method_name(stop) == "len":- inner = [c for c in stop.child_by_field_name("arguments").children if c.is_named]- if len(inner) == 1:- return self._source(inner[0], loop, probe, hop=hop)- return "unknown", None-- def reads_config_table(self, call: Node, nodes: list[Node]) -> bool:- """Every SQL text and selected model names a registry or configuration table."""- walked = [n for part in nodes for n in self._walk(part)]- sql = [- n.text or b""- for n in walked- if n.type == "string" and _PY_SQL_LITERAL_RE.match(_text(n))- ]- models = [- model- for n in walked- if n.type == "call" and self.callee_method_name(n) in ("select", "query")- for model in self._selected_models(n)- ]- return bool(sql or models) and (- all(lex.is_config_sql(text) for text in sql)- and all(lex.is_config_model(model) for model in models)- )-- def _selected_models(self, select: Node) -> list[str]:- """``select(Repository)`` / ``query(Repository.id, Repository.path)`` -> the model- names; ``func.count()`` and other expressions name none."""- args = select.child_by_field_name("arguments")- models: list[str] = []- for arg in (c for c in args.children if c.is_named) if args is not None else ():- path = self._dotted_path(arg) or ""- head = next((seg for seg in path.split(".") if _PY_MODEL_NAME_RE.match(seg)), None)- if head is not None:- models.append(head)- return models-- def _literal_constant(self, expr: Node, name: bytes) -> bool:- """``self._TABLES`` where the file binds ``_TABLES`` once, to a literal collection."""- text = name.decode("utf-8", "replace")- if len(text.strip("_")) < 2 or not text.isupper():- return False- root = expr- while root.parent is not None:- root = root.parent- rights = [- n.child_by_field_name("right")- for n in self._walk(root)- if n.type == "assignment"- and (left := n.child_by_field_name("left")) is not None- and left.type == "identifier"- and left.text == name- ]- return (- len(rights) == 1- and rights[0] is not None- and rights[0].type in _PY_CONSTANT_RHS- and all(_scalar_literal(item) for item in rights[0].named_children)- )-- def _magnitude(- self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool- ) -> LoopMagnitude | None:- """Grows (a query result or a directory listing), bounded, or ``None``.-- *hop* allows one step through the name's last assignment before the- loop; a parameter has none and stays unknown on purpose.- """- if expr is None:- return None- if expr.type in ("await", "parenthesized_expression"):- inner = next((c for c in expr.children if c.is_named), None)- return self._magnitude(inner, loop, probe, hop=hop)- if expr.type == "identifier" and hop and expr.text:- rhs = self.reaching_value(loop, expr.text)- return self._magnitude(rhs, loop, probe, hop=False)- if expr.type == "subscript":- return self._slice_magnitude(expr)- if expr.type == "boolean_operator" and any(c.type == "or" for c in expr.children):- # ``(await q.execute()).data or []``: the fallback is empty, the read is not.- left = expr.child_by_field_name("left")- return self._grows(self._magnitude(left, loop, probe, hop=hop))- if expr.type == "attribute":- attr = expr.child_by_field_name("attribute")- if attr is None or (attr.text or b"").decode() not in _PY_RESULT_PROJECTIONS:- return None- return self._grows(self._magnitude(expr.child_by_field_name("object"), loop, probe, hop=hop))- return self._call_magnitude(expr, loop, probe, hop=hop) if expr.type == "call" else None-- def _call_magnitude(- self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool- ) -> LoopMagnitude | None:- method = self.callee_method_name(expr) or ""- member = self.callee_is_attribute(expr)- if method == "range" and not member:- return self._range_magnitude(expr, loop, probe, hop=hop)- if probe(expr) in ("db", "network"):- # A read capped in the query is as large as its cap; a statement or result- # built before the loop (``stmt = ....limit(n)``) is read one assignment back.- names = [n.text for n in self._walk(expr) if n.type == "identifier"]- parts = [expr, *filter(None, (self.reaching_value(loop, name) for name in names))]- capped = any(self.is_cap(n, loop) for part in parts for n in self._walk(part))- return None if capped else "grows_with_data"- if member and method in _PY_RESULT_PROJECTIONS:- receiver = expr.child_by_field_name("function").child_by_field_name("object")- return self._grows(self._magnitude(receiver, loop, probe, hop=hop))- return None-- def is_cap(self, node: Node, loop: Node) -> bool:- """``.limit(n)`` and peers, a SQL string saying ``LIMIT n``, or a filter to a- set of keys (:meth:`read_key_column`), which is as large as the keys are."""- if node.type == "string":- return bool(_PY_SQL_LIMIT_RE.search(_PY_SQL_COMMENT_RE.sub(b"", node.text or b"")))- method = self.callee_method_name(node) if node.type == "call" else None- if method in _PY_READ_CAPS:- return True- return node.type == "call" and self.read_key_column(node, loop) is not None-- def read_key_column(self, call: Node, anchor: Node) -> str | None:- """``Model.col.in_(keys)`` and PostgREST ``.in_("col", keys)`` -> ``col``;- ``filter_by(id=k)`` and ``.eq("id", k)`` -> ``id``. Keys a query yields- (``.in_(select(...))``, or a name bound to one) grow with that table."""- method = self.callee_method_name(call)- arg_list = call.child_by_field_name("arguments")- args = arg_list.named_children if arg_list is not None else []- if method == "filter_by":- named = {_field_text(a, "name") for a in args if a.type == "keyword_argument"}- return "id" if "id" in named else None- if method == "eq":- return "id" if len(args) == 2 and _string_value(args[0]) == "id" else None- found = _in_filter(call, args) if method == "in_" else None- if found is None or not self._handed_in(found[1], anchor):- return None- return found[0]-- def _handed_in(self, keys: Node, anchor: Node) -> bool:- """*keys* holds values given to the code, not ones a query yields or a fixed- set of states (``in_(("a", "b"))``, ``in_(SOURCES)``) whose rows grow."""- values = [- self.reaching_value(anchor, ref.text or b"") if ref.type == "identifier" else None- for ref in _value_refs(keys)- ]- if all(_fixed_set(value) for value in values):- return False- parts = [keys, *filter(None, values)]- return not any(- n.type == "call" and self.callee_method_name(n) in _PY_KEY_READS- for part in parts- for n in self._walk(part)- )-- def _range_magnitude(- self, call: Node, loop: Node, probe: SinkProbe, *, hop: bool- ) -> LoopMagnitude | None:- """``range(len(rows))`` grows with ``rows``; ``range(3)`` and ``range(MAX)``- with ``MAX = 3`` bound once in the module are bounded."""- args = [c for c in call.child_by_field_name("arguments").children if c.is_named]- if not args:- return None- stop = args[1] if len(args) > 1 else args[0]- if stop.type == "call" and self.callee_method_name(stop) == "len":- inner = [c for c in stop.child_by_field_name("arguments").children if c.is_named]- if len(inner) == 1:- return self._grows(self._magnitude(inner[0], loop, probe, hop=hop))- return None- # A name alone (``MAX_RETRIES``, ``self.max_attempts``) proves nothing.- return "bounded" if stop.type == "integer" or _module_int_constant(stop) else None-- def _slice_magnitude(self, subscript: Node) -> LoopMagnitude | None:- """``xs[:5]`` / ``xs[:LIMIT]`` / ``xs[i:i + 4]``: a constant-width slice."""- piece = subscript.child_by_field_name("subscript")- if piece is None or piece.type != "slice" or sum(c.type == ":" for c in piece.children) != 1:- return None- colon = next(i for i, c in enumerate(piece.children) if c.type == ":")- start = next((c for c in piece.children[:colon] if c.is_named), None)- stop = next((c for c in piece.children[colon + 1 :] if c.is_named), None)- if stop is None:- return None- if start is None:- return "bounded" if stop.type == "integer" or _module_int_constant(stop) else None- if stop.type == "binary_operator":- left, right = stop.child_by_field_name("left"), stop.child_by_field_name("right")- same_start = left is not None and self._dotted_path(left) == self._dotted_path(start)- if same_start and right is not None and right.type == "integer":- return "bounded"- return None-- def batch_form(self, sink: Node, loop: Node, probe: SinkProbe) -> BatchForm | None:- target = loop.child_by_field_name("left")- if loop.type != "for_statement" or target is None or target.type != "identifier":- return None- body = self.loop_body(loop) or loop- iterable = self.loop_iterable_name(loop)- for node in self._walk(body):- if (- node.type == "call"- and self.callee_is_attribute(node)- and self.callee_root_name(node) == iterable- and self.callee_method_name(node) in self.sequence_appends- ):- return None # a worklist: its keys are not known before the loop- refs = [n for n in self._walk(sink) if n.type == "identifier" and n.text == target.text]- methods = {self.callee_method_name(n) for n in self._walk(sink) if n.type == "call"}- if len(refs) != 1 or methods & _PY_WRITES or self._reads_loop_local(sink, body):- return None- call = self._keyed_filter(sink, refs[0], methods)- if call is None:- return None- if "delete" in methods:- # Batched, a delete also removes rows a skipped iteration would have left.- kept = self._unconditional(sink, body)- else:- kept = not self._builds_in_order(body)- if self._orm_get(sink) is not None:- # ``get`` answers from the session's identity map when the row is already- # loaded, which no syntax shows: held out, half of these sites made no- # round trip. The bulk form is named, never proven.- kept = False- equivalent = (- kept- and not methods & _PY_PER_KEY_LIMITS- and not self._limited_downstream(sink, body)- and self._only_io_in_body(sink, body, probe)- and not self._calls_for_effect(sink, body)- )- return BatchForm(call, equivalent)-- def _calls_for_effect(self, sink: Node, body: Node) -> bool:- """Another statement calls something this function did not build as a list, set- or dict (``run_task(doc)``, ``session.add(r)``, ``t = task.delay(d)``), which may- write what the batched read would read before it ran."""- for node in self._walk(body, prune=_PY_SCOPES):- if node.type == "expression_statement":- call, assigned = node.named_children[0], False- elif node.type == "assignment":- call, assigned = node.child_by_field_name("right"), True- else:- continue- if call is not None and call.type == "await":- call = next((c for c in call.children if c.is_named), None)- if call is None or call.type != "call" or self._within(call, sink):- continue- if self._has_effect(call, assigned=assigned):- return True- return False-- def _has_effect(self, call: Node, *, assigned: bool) -> bool:- """Not ``seen.add(x)`` on a local collection, and, when its result is kept, not a- builtin (``str(r.id)``) or a method on a local (``res.first()``)."""- if not self.callee_is_attribute(call):- return not (assigned and self.callee_method_name(call) in _PY_BUILTINS)- before = self.bindings_before(call, (self.callee_root_name(call) or "").encode())- if not before:- return True- rhs = before[-1][1]- return not (assigned or self._rhs_is_list(rhs) or self._rhs_is_nonlist_container(rhs))-- def _limited_downstream(self, sink: Node, body: Node) -> bool:- """The result is cut to one row after the call (``(await q).first()``,- ``res = await q`` then ``res.scalar_one()``), which one IN query does not do."""- names: set[bytes | None] = set()- cur = sink- while cur.parent is not None and cur.parent != body:- cur = cur.parent- if cur.type == "call" and self.callee_method_name(cur) in _PY_ONE_ROW:- return True- left = cur.child_by_field_name("left") if cur.type == "assignment" else None- if left is not None and left.type == "identifier":- names.add(left.text)- return any(- n.type == "call"- and self.callee_method_name(n) in _PY_ONE_ROW- and (self.callee_root_name(n) or "").encode() in names- for n in self._walk(body)- )-- def _keyed_filter(self, sink: Node, ref: Node, methods: set[str | None]) -> str | None:- """The bulk filter when *sink* selects or deletes by equality on *ref*."""- orm_get = self._orm_get(sink)- if orm_get is not None and self._is_key_of(orm_get[1], ref):- return f"select({orm_get[0]}).where(inspect({orm_get[0]}).primary_key[0].in_(keys))"- for node in self._walk(sink):- if node.type == "call" and self.callee_method_name(node) == "eq":- # supabase-py: ``.table(t).select(...).eq("col", key)``- args = [c for c in node.child_by_field_name("arguments").children if c.is_named]- if (- len(args) == 2- and args[0].type == "string"- and self._is_key_of(args[1], ref)- and ("select" in methods) != ("delete" in methods)- ):- column = next(- (c.text.decode() for c in args[0].children if c.type == "string_content"),- None,- )- return f'.in_("{column}", keys)' if column else None- if node.type == "comparison_operator" and any(c.type == "==" for c in node.children):- # SQLAlchemy: ``.where(M.col == key)`` / ``.filter(M.col == key)``- sides = [c for c in node.children if c.is_named]- if len(sides) != 2:- continue- for column, key in (sides, sides[::-1]):- path = self._dotted_path(column)- if path and "." in path and self._is_key_of(key, ref):- return f"{path}.in_(keys)"- return None-- def concurrency_bound(self, sink: Node, loop: Node) -> str | None:- if self._exits_early(self.loop_body(loop) or loop):- return None- cur = sink.parent- while cur is not None and cur != loop:- if cur.type == "with_statement" and any(c.type == "async" for c in cur.children):- clause = next((c for c in cur.children if c.type == "with_clause"), None)- for item in clause.children if clause is not None else ():- value = item.child_by_field_name("value") if item.type == "with_item" else None- # A name, not a constructor: ``async with Semaphore(5)`` in the- # loop makes a fresh bound per iteration, which bounds nothing.- path = self._dotted_path(value) if value is not None else None- if path and _PY_SEM_NAME_RE.search(path.rsplit(".", 1)[-1]):- return path- cur = cur.parent- return None-DIALECT = PythonPerfDialect()diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_batch.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_batch.pynew file mode 100644index 000000000..05e0c0bbe--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_batch.py@@ -0,0 +1,188 @@+"""Whether a per-item Python query in a loop has a single batched form."""++from __future__ import annotations++from typing import TYPE_CHECKING++from ..loop_facts import BatchForm+from .python_lexicon import (+ _PY_BUILTINS,+ _PY_ONE_ROW,+ _PY_PER_KEY_LIMITS,+ _PY_SCOPES,+ _PY_SEM_NAME_RE,+ _PY_WRITES,+)+from .python_nodes import _string_content++if TYPE_CHECKING:+ from tree_sitter import Node++ from ..loop_facts import SinkProbe+++class PythonBatchForms:+ """Batched forms and concurrency bounds. Mixed into :class:`~.python.PythonPerfDialect`."""++ def batch_form(self, sink: Node, loop: Node, probe: SinkProbe) -> BatchForm | None:+ target = loop.child_by_field_name("left")+ if loop.type != "for_statement" or target is None or target.type != "identifier":+ return None+ body = self.loop_body(loop) or loop+ if self._appends_to_iterable(body, self.loop_iterable_name(loop)):+ return None # a worklist: its keys are not known before the loop+ methods = {self.callee_method_name(n) for n in self._walk(sink) if n.type == "call"}+ call = self._batch_call(sink, body, target, methods)+ if call is None:+ return None+ return BatchForm(call, self._batch_equivalent(sink, body, probe, methods))++ def _appends_to_iterable(self, body: Node, iterable: str | None) -> bool:+ return any(+ node.type == "call"+ and self.callee_is_attribute(node)+ and self.callee_root_name(node) == iterable+ and self.callee_method_name(node) in self.sequence_appends+ for node in self._walk(body)+ )++ def _batch_call(+ self, sink: Node, body: Node, target: Node, methods: set[str | None]+ ) -> str | None:+ """The bulk filter for a read keyed once on the loop target and nothing loop-local."""+ refs = [n for n in self._walk(sink) if n.type == "identifier" and n.text == target.text]+ if len(refs) != 1 or methods & _PY_WRITES or self._reads_loop_local(sink, body):+ return None+ return self._keyed_filter(sink, refs[0], methods)++ def _batch_equivalent(+ self, sink: Node, body: Node, probe: SinkProbe, methods: set[str | None]+ ) -> bool:+ if "delete" in methods:+ # Batched, a delete also removes rows a skipped iteration would have left.+ kept = self._unconditional(sink, body)+ else:+ kept = not self._builds_in_order(body)+ if self._orm_get(sink) is not None:+ # ``get`` may answer from the session's identity map, which no syntax+ # shows, so the bulk form is named, never proven.+ kept = False+ return (+ kept+ and not methods & _PY_PER_KEY_LIMITS+ and not self._limited_downstream(sink, body)+ and self._only_io_in_body(sink, body, probe)+ and not self._calls_for_effect(sink, body)+ )++ def _calls_for_effect(self, sink: Node, body: Node) -> bool:+ """Another statement calls something this function did not build as a list, set+ or dict (``run_task(doc)``, ``session.add(r)``, ``t = task.delay(d)``), which may+ write what the batched read would read before it ran."""+ for node in self._walk(body, prune=_PY_SCOPES):+ if node.type == "expression_statement":+ call, assigned = node.named_children[0], False+ elif node.type == "assignment":+ call, assigned = node.child_by_field_name("right"), True+ else:+ continue+ if call is not None and call.type == "await":+ call = next((c for c in call.children if c.is_named), None)+ if call is None or call.type != "call" or self._within(call, sink):+ continue+ if self._has_effect(call, assigned=assigned):+ return True+ return False++ def _has_effect(self, call: Node, *, assigned: bool) -> bool:+ """Not ``seen.add(x)`` on a local collection, and, when its result is kept, not a+ builtin (``str(r.id)``) or a method on a local (``res.first()``)."""+ if not self.callee_is_attribute(call):+ return not (assigned and self.callee_method_name(call) in _PY_BUILTINS)+ before = self.bindings_before(call, (self.callee_root_name(call) or "").encode())+ if not before:+ return True+ rhs = before[-1][1]+ return not (assigned or self._rhs_is_list(rhs) or self._rhs_is_nonlist_container(rhs))++ def _limited_downstream(self, sink: Node, body: Node) -> bool:+ """The result is cut to one row after the call (``(await q).first()``,+ ``res = await q`` then ``res.scalar_one()``), which one IN query does not do."""+ names: set[bytes | None] = set()+ cur = sink+ while cur.parent is not None and cur.parent != body:+ cur = cur.parent+ if cur.type == "call" and self.callee_method_name(cur) in _PY_ONE_ROW:+ return True+ left = cur.child_by_field_name("left") if cur.type == "assignment" else None+ if left is not None and left.type == "identifier":+ names.add(left.text)+ return any(+ n.type == "call"+ and self.callee_method_name(n) in _PY_ONE_ROW+ and (self.callee_root_name(n) or "").encode() in names+ for n in self._walk(body)+ )++ def _keyed_filter(self, sink: Node, ref: Node, methods: set[str | None]) -> str | None:+ """The bulk filter when *sink* selects or deletes by equality on *ref*."""+ orm_get = self._orm_get(sink)+ if orm_get is not None and self._is_key_of(orm_get[1], ref):+ return f"select({orm_get[0]}).where(inspect({orm_get[0]}).primary_key[0].in_(keys))"+ for node in self._walk(sink):+ if node.type == "call" and self.callee_method_name(node) == "eq":+ column_arg = self._eq_column_arg(node, ref, methods)+ if column_arg is not None:+ column = _string_content(column_arg)+ return f'.in_("{column}", keys)' if column else None+ if node.type == "comparison_operator" and any(c.type == "==" for c in node.children):+ found = self._equality_filter(node, ref)+ if found is not None:+ return found+ return None++ def _eq_column_arg(self, call: Node, ref: Node, methods: set[str | None]) -> Node | None:+ """The column string of supabase-py ``.table(t).select(...).eq("col", key)``."""+ args = [c for c in call.child_by_field_name("arguments").children if c.is_named]+ if (+ len(args) == 2+ and args[0].type == "string"+ and self._is_key_of(args[1], ref)+ and ("select" in methods) != ("delete" in methods)+ ):+ return args[0]+ return None++ def _equality_filter(self, comparison: Node, ref: Node) -> str | None:+ """SQLAlchemy ``.where(M.col == key)`` / ``.filter(M.col == key)``."""+ sides = [c for c in comparison.children if c.is_named]+ if len(sides) != 2:+ return None+ for column, key in (sides, sides[::-1]):+ path = self._dotted_path(column)+ if path and "." in path and self._is_key_of(key, ref):+ return f"{path}.in_(keys)"+ return None++ def concurrency_bound(self, sink: Node, loop: Node) -> str | None:+ if self._exits_early(self.loop_body(loop) or loop):+ return None+ cur = sink.parent+ while cur is not None and cur != loop:+ if cur.type == "with_statement" and any(c.type == "async" for c in cur.children):+ path = self._semaphore_entered(cur)+ if path is not None:+ return path+ cur = cur.parent+ return None++ def _semaphore_entered(self, with_stmt: Node) -> str | None:+ """The semaphore-like name an ``async with`` enters. A constructor+ (``async with Semaphore(5)``) is fresh per iteration and bounds nothing."""+ clause = next((c for c in with_stmt.children if c.type == "with_clause"), None)+ for item in clause.children if clause is not None else ():+ value = item.child_by_field_name("value") if item.type == "with_item" else None+ path = self._dotted_path(value) if value is not None else None+ if path and _PY_SEM_NAME_RE.search(path.rsplit(".", 1)[-1]):+ return path+ return Nonediff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.pynew file mode 100644index 000000000..6e7d4b2c8--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py@@ -0,0 +1,187 @@+"""Python call, method and node-kind vocabularies the perf dialect matches on."""++from __future__ import annotations++import builtins+import re++# Network round-trip verbs (shared across languages that name HTTP methods).+HTTP_VERBS: frozenset[str] = frozenset(+ {"get", "post", "put", "delete", "patch", "head", "options", "request", "send", "stream"}+)++# Python DBAPI / SQLAlchemy execution sinks, split into three strata so the+# ambiguous accessors can be gated harder than the unambiguous ones (see+# ``sink_kind``).+PY_DB_UNAMBIGUOUS: frozenset[str] = frozenset({"fetchone", "fetchall", "fetchmany"})+# DB verbs other clients spell too: GitPython's ``repo.commit()``, an HTTP or+# GraphQL SDK's ``request.execute()``, a builder's ``.scalar()``.+PY_DB_NEEDS_EVIDENCE: frozenset[str] = frozenset(+ {"commit", "execute", "executemany", "scalars", "scalar", "scalar_one", "scalar_one_or_none"}+)+# Without a db import, the call itself can still show it runs a query: a+# PostgREST chain (``client.table("t")...execute()``) or a statement argument+# (``session.execute(select(...))``, ``cur.execute("SELECT ...")``).+_PY_QUERY_CHAIN_METHODS: frozenset[str] = frozenset({"table", "from_"})+_PY_STATEMENT_BUILDERS: frozenset[str] = frozenset({"select", "insert", "update", "delete", "text"})+_PY_SQL_LITERAL_RE = re.compile(+ r"^[rRuUbBfF]*[\"']+\s*(?:SELECT|INSERT|UPDATE|DELETE|WITH|CREATE|DROP|ALTER|PRAGMA)\b",+ re.IGNORECASE,+)+PY_DB_AMBIGUOUS: frozenset[str] = frozenset({"all", "first", "one", "one_or_none"})+PY_SUBPROC_METHODS: frozenset[str] = frozenset(+ {"run", "call", "check_call", "check_output", "Popen"}+)+# Any of these anywhere in a read's call chain proves it is bounded, gating+# ``unbounded_read_reduced_in_memory`` (analysis.health.perf.unbounded_reduction).+PY_UNBOUNDED_READ_BOUND_METHODS: frozenset[str] = frozenset(+ {"limit", "range", "single", "maybe_single", "first", "count", "head", "aggregate"}+)++# Matched on the method name alone: only ``pathlib.Path`` spells these, and the+# receiver is usually a local (``src_path.read_text()``) that ``io_names`` cannot+# classify. Metadata calls (``exists`` / ``stat`` / ``glob``) are cheap and left out.+PY_FS_METHODS: frozenset[str] = frozenset(+ {"read_text", "read_bytes", "write_text", "write_bytes"}+)+# ``os`` / ``shutil`` verbs, gated on the literal module root. Ceiling: the root is+# the leftmost name of the whole chain, so ``os.path.join(a).verb()`` lands here too;+# no verb a ``str`` also has (``replace``) may join. Upgrade: require two segments.+PY_OS_FS_METHODS: frozenset[str] = frozenset(+ {+ "walk",+ "listdir",+ "scandir",+ "remove",+ "unlink",+ "rename",+ "makedirs",+ "mkdir",+ "rmdir",+ "removedirs",+ }+)+PY_SHUTIL_FS_METHODS: frozenset[str] = frozenset(+ {+ "copy",+ "copy2",+ "copyfile",+ "copytree",+ "move",+ "rmtree",+ "make_archive",+ "unpack_archive",+ }+)++# Python string-literal node kinds (f-strings parse as ``string`` too).+_PY_STRING_KINDS: frozenset[str] = frozenset({"string", "concatenated_string"})+_PY_AUG_ASSIGN_KINDS: frozenset[str] = frozenset({"augmented_assignment"})++# Connection / client / pool constructors, as ``(root, method)`` on a distinctive I/O library ...+_PY_RESOURCE_CTORS: frozenset[tuple[str, str]] = frozenset(+ {+ ("sqlite3", "connect"),+ ("psycopg", "connect"),+ ("psycopg2", "connect"),+ ("pymysql", "connect"),+ ("MySQLdb", "connect"),+ ("aiosqlite", "connect"),+ ("httpx", "Client"),+ ("httpx", "AsyncClient"),+ ("requests", "Session"),+ ("aiohttp", "ClientSession"),+ ("boto3", "client"),+ ("boto3", "resource"),+ ("redis", "Redis"),+ ("redis", "StrictRedis"),+ ("pymongo", "MongoClient"),+ }+)+# ... plus a few constructors distinctive enough on their own (the rightmost+# name), so an aliased ``from sqlalchemy import create_engine`` still resolves.+_PY_RESOURCE_METHODS: frozenset[str] = frozenset({"create_engine", "MongoClient"})++# Lock acquisition only; ``release`` would count the same critical section twice.+_PY_LOCK_METHODS: frozenset[str] = frozenset({"acquire"})++# RHS node kinds that prove a name is bound to a list (the membership gate).+_PY_LIST_RHS_KINDS: frozenset[str] = frozenset({"list", "list_comprehension"})+# Builtins whose call result is provably a list.+_PY_LIST_BUILTINS: frozenset[str] = frozenset({"list", "sorted"})+# RHS shapes of a set or dict: a name bound to one anywhere in the file leaves the membership gate.+_PY_NONLIST_RHS_KINDS: frozenset[str] = frozenset(+ {"set", "dictionary", "set_comprehension", "dictionary_comprehension"}+)+_PY_NONLIST_BUILTINS: frozenset[str] = frozenset({"set", "dict", "frozenset"})++# Promotion facts (perf/loop_facts.py). Calls and attributes that read a query+# result out without changing how many rows it holds.+_PY_RESULT_PROJECTIONS: frozenset[str] = frozenset(+ {"all", "scalars", "fetchall", "json", "data", "rows"}+)+# Raw SQL capped in its own text: ``sa.text("... LIMIT 1000")``.+_PY_SQL_LIMIT_RE = re.compile(rb"(?is)\bselect\b.*\blimit\s+\d+\b")+_PY_SQL_COMMENT_RE = re.compile(rb"--[^\n]*|/\*.*?\*/", re.S)+# A per-key call that limits or orders its rows is not the read one IN query makes.+_PY_PER_KEY_LIMITS: frozenset[str] = frozenset(+ {+ "limit", "single", "maybe_single", "first", "one", "scalar_one",+ "scalar_one_or_none", "order", "order_by", "range", "offset",+ }+)+_PY_ONE_ROW: frozenset[str] = _PY_PER_KEY_LIMITS | {"scalar"}+_PY_READ_CAPS: frozenset[str] = _PY_ONE_ROW - {"order", "order_by", "offset"}+_PY_WRITES: frozenset[str] = frozenset({"update", "insert", "upsert", "values"})+# Keys a query yields (``.in_(select(...))``) are as many as that table's rows.+_PY_KEY_READS: frozenset[str] = _PY_STATEMENT_BUILDERS | {+ "subquery", "scalar_subquery", "query", "cte", "alias", "union", "union_all",+ "execute", "table", "from_",+}+_PY_SCOPES: frozenset[str] = frozenset({"function_definition", "lambda", "class_definition"})+_PY_BUILTINS: frozenset[str] = frozenset(dir(builtins))+_PY_SEM_NAME_RE = re.compile(r"(?i)(sem|semaphore|limiter|limit)$")+# A model class is CapWords; an ALL_CAPS constant (``API_URL``) is not.+_PY_MODEL_NAME_RE = re.compile(r"^[A-Z]\w*[a-z]\w*$")+# Directory listings; the recursive ones walk a whole tree.+_PY_LISTINGS: frozenset[str] = frozenset(+ {+ "glob", "iglob", "rglob", "iterdir", "walk", "listdir", "scandir", "traverse",+ "iter_glob", "glob_via", "walk_repo", "_rglob",+ }+)+_PY_RECURSIVE_LISTINGS: frozenset[str] = frozenset(+ {"rglob", "walk", "traverse", "walk_repo", "_rglob"}+)+# Reads that yield lines or chunks as they arrive.+_PY_STREAMS: frozenset[str] = frozenset(+ {+ "readlines", "iter_lines", "iter_content", "iter_bytes", "iter_text",+ "aiter_lines", "aiter_bytes", "aiter_text", "aiter_raw",+ }+)+# Wrappers and chunkers whose first argument is what the loop walks.+_PY_PASS_THROUGH: frozenset[str] = frozenset(+ {+ "enumerate", "sorted", "reversed", "list", "set", "tuple", "iter", "zip",+ "batched", "chunked", "ichunked", "chunks", "iter_chunks", "grouper",+ }+)+_PY_CONSTANT_RHS: frozenset[str] = frozenset({"list", "tuple", "set", "dictionary"})+++_PY_DEF_KINDS = frozenset({"function_definition", "lambda", "class_definition"})+# Nodes an identifier sits in when it is the name being bound, not read.+_PY_BINDING_PARENTS = frozenset(+ {+ "global_statement", "nonlocal_statement", "as_pattern_target", "aliased_import",+ "import_from_statement", "import_statement", "parameters", "lambda_parameters",+ "named_expression",+ }+)+_PY_BINDING_WRAPPERS = frozenset(+ {+ "pattern_list", "tuple_pattern", "list_pattern", "typed_parameter", "default_parameter",+ "typed_default_parameter", "list_splat_pattern", "dotted_name",+ }+)diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.pynew file mode 100644index 000000000..9d51e6da2--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py@@ -0,0 +1,193 @@+"""Tree-sitter helpers for Python syntax the perf dialect reads."""++from __future__ import annotations++from typing import TYPE_CHECKING++from .base import BasePerfDialect+from .python_lexicon import (+ _PY_BINDING_PARENTS,+ _PY_BINDING_WRAPPERS,+ _PY_CONSTANT_RHS,+ _PY_DEF_KINDS,+)++if TYPE_CHECKING:+ from tree_sitter import Node+++def _binds(ident: Node) -> bool:+ """Whether *ident* is a name being bound (assigned, imported, a parameter)."""+ cur, parent = ident, ident.parent+ while parent is not None and parent.type in _PY_BINDING_WRAPPERS:+ cur, parent = parent, parent.parent+ if parent is None:+ return False+ if parent.type in ("assignment", "augmented_assignment", "for_statement", "for_in_clause"):+ return parent.child_by_field_name("left") == cur+ if parent.type in ("function_definition", "class_definition"):+ return parent.child_by_field_name("name") == cur+ return parent.type in _PY_BINDING_PARENTS+++def _bindings(scope: Node, name: bytes) -> list[Node]:+ """Identifiers binding *name* in *scope* itself, not in a nested scope."""+ found: list[Node] = []+ stack = list(scope.children)+ while stack:+ node = stack.pop()+ if node.type in _PY_DEF_KINDS:+ stack.extend(c for c in (node.child_by_field_name("name"),) if c is not None)+ continue+ if node.type == "identifier" and node.text == name and _binds(node):+ found.append(node)+ stack.extend(node.children)+ return found+++def _module_int_constant(name: Node) -> bool:+ """An ALL_CAPS name its own module binds exactly once, to an integer literal+ (``MAX = 5``), and no enclosing function rebinds or shadows.++ The spelling alone proves nothing (``NUM_USERS`` grows), so a name rebound,+ imported, augmented, written through ``global``, shadowed by a parameter or+ a local, or bound to anything but a literal stays unknown.+ """+ text = name.text or b""+ if name.type != "identifier" or len(text) < 2 or not text.decode().isupper():+ return False+ module = _unshadowed_module(name, text)+ if module is None:+ return False+ bound = _bindings(module, text)+ if len(bound) != 1 or bound[0].parent is None or bound[0].parent.type != "assignment":+ return False+ right = bound[0].parent.child_by_field_name("right")+ return right is not None and right.type == "integer"+++def _unshadowed_module(name: Node, text: bytes) -> Node | None:+ """The module root, when no enclosing function binds *text* and no ``global`` names it."""+ scope = name.parent+ while scope is not None and scope.parent is not None:+ if scope.type in ("function_definition", "lambda") and _bindings(scope, text):+ return None+ scope = scope.parent+ if scope is None or any(+ n.type == "global_statement" and any(c.text == text for c in n.named_children)+ for n in BasePerfDialect._walk(scope)+ ):+ return None+ return scope+++def _int_literal(node: Node) -> bool:+ return node.type == "integer" or (+ node.type == "unary_operator" and any(c.type == "integer" for c in node.children)+ )+++def _literal_range(call: Node) -> bool:+ """``range(...)`` whose every argument is an integer literal; ``range(len(x))`` is data-dependent."""+ fn = call.child_by_field_name("function")+ if fn is None or (fn.text or b"").decode("utf-8", "replace") != "range":+ return False+ args = call.child_by_field_name("arguments")+ if args is None:+ return False+ return all(_int_literal(a) for a in args.children if a.is_named)+++def _is_or(expr: Node) -> bool:+ return expr.type == "boolean_operator" and any(c.type == "or" for c in expr.children)+++def _slice_bounds(subscript: Node) -> tuple[Node | None, Node] | None:+ """``(start, stop)`` of a one-colon slice that has a stop."""+ piece = subscript.child_by_field_name("subscript")+ if piece is None or piece.type != "slice" or sum(c.type == ":" for c in piece.children) != 1:+ return None+ colon = next(i for i, c in enumerate(piece.children) if c.type == ":")+ start = next((c for c in piece.children[:colon] if c.is_named), None)+ stop = next((c for c in piece.children[colon + 1 :] if c.is_named), None)+ return None if stop is None else (start, stop)+++def _string_content(string: Node) -> str | None:+ return next((c.text.decode() for c in string.children if c.type == "string_content"), None)+++def _text(node: Node) -> str:+ return (node.text or b"").decode("utf-8", "replace")+++def _field_text(node: Node, field: str) -> str:+ child = node.child_by_field_name(field)+ return _text(child) if child is not None else ""+++def _value_refs(expr: Node) -> list[Node]:+ """The names and dotted paths *expr* reads values from: not a callee+ (``tuple`` in ``tuple(xs)``), a keyword, or an ALL_CAPS constant."""+ refs = []+ stack = [expr]+ while stack:+ node = stack.pop()+ if node.type not in ("identifier", "attribute"):+ stack.extend(node.children)+ continue+ parent = node.parent+ callee = parent is not None and parent.child_by_field_name("function") == node+ keyword = parent is not None and parent.type == "keyword_argument" and (+ parent.child_by_field_name("name") == node+ )+ last = node.child_by_field_name("attribute") if node.type == "attribute" else node+ constant = last is not None and _text(last).strip("_").isupper()+ if callee and node.type == "attribute":+ # ``self.ids.copy()`` still reads ``self.ids``.+ stack.extend(filter(None, [node.child_by_field_name("object")]))+ elif not (callee or keyword or constant):+ refs.append(node)+ return refs+++def _string_value(node: Node) -> str | None:+ return _text(node).strip("\"'") if node.type == "string" else None+++def _in_filter(call: Node, args: list[Node]) -> tuple[str, Node] | None:+ """``(column, keys)`` of ``Model.col.in_(keys)`` or PostgREST ``.in_("col", keys)``."""+ if len(args) == 2:+ column = _string_value(args[0])+ return (column, args[1]) if column else None+ fn = call.child_by_field_name("function")+ receiver = fn.child_by_field_name("object") if fn is not None else None+ if len(args) != 1 or receiver is None or receiver.type != "attribute":+ return None+ column = _field_text(receiver, "attribute")+ return (column, args[0]) if column else None+++def _fixed_set(value: Node | None) -> bool:+ """A non-empty literal of scalars; an empty one is a container filled later."""+ return (+ value is not None+ and value.type in _PY_CONSTANT_RHS+ and bool(value.named_children)+ and all(_scalar_literal(item) for item in value.named_children)+ )+++_PY_SCALARS = frozenset({"string", "concatenated_string", "integer", "float", "true", "false", "none"})+++def _scalar_literal(item: Node) -> bool:+ """``"a"`` / ``3`` / ``("a", 1)`` / ``"k": v`` by its key: an entry the source+ spells out. A name or call in the literal (``[ROOT / "a", repo.path]``) may+ stand for anything, so the table is not shown to be a fixed set."""+ if item.type == "pair":+ key = item.child_by_field_name("key")+ return key is not None and key.type in _PY_SCALARS+ if item.type == "tuple":+ return all(child.type in _PY_SCALARS for child in item.named_children)+ return item.type in _PY_SCALARSdiff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_sources.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_sources.pynew file mode 100644index 000000000..7a947e9a7--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_sources.py@@ -0,0 +1,350 @@+"""What a Python loop walks, and how large it can grow."""++from __future__ import annotations++from typing import TYPE_CHECKING++from .. import loop_lexicon as lex+from ..loop_facts import LoopSource+from .python_lexicon import (+ _PY_CONSTANT_RHS,+ _PY_KEY_READS,+ _PY_LISTINGS,+ _PY_MODEL_NAME_RE,+ _PY_PASS_THROUGH,+ _PY_READ_CAPS,+ _PY_RESULT_PROJECTIONS,+ _PY_SQL_COMMENT_RE,+ _PY_SQL_LIMIT_RE,+ _PY_SQL_LITERAL_RE,+ _PY_STREAMS,+)+from .python_nodes import (+ _field_text,+ _fixed_set,+ _in_filter,+ _is_or,+ _module_int_constant,+ _scalar_literal,+ _slice_bounds,+ _string_value,+ _text,+ _value_refs,+)++if TYPE_CHECKING:+ from tree_sitter import Node++ from ..loop_facts import LoopIterates, LoopMagnitude, SinkProbe++ Source = tuple[LoopIterates, LoopMagnitude | None]+++class PythonLoopSources:+ """Loop sources and magnitudes. Mixed into :class:`~.python.PythonPerfDialect`."""++ def loop_source(self, loop: Node, probe: SinkProbe) -> LoopSource | None:+ if loop.type != "for_statement":+ return None+ right = loop.child_by_field_name("right")+ baseline = self._magnitude(right, loop, probe, hop=True) or "unknown"+ iterates, claim = self._source(right, loop, probe, hop=True)+ if iterates == "unknown" and baseline == "unknown":+ return None+ return LoopSource(iterates, claim or baseline)++ def _source(self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:+ """What *expr* walks, and the magnitude that settles when it differs from+ what the iterable's syntax alone proves (``None``: keep that).++ *hop* allows one step through the name's last assignment before the loop.+ """+ if expr is None:+ return "unknown", None+ if expr.type in ("await", "parenthesized_expression"):+ inner = next((c for c in expr.children if c.is_named), None)+ return self._source(inner, loop, probe, hop=hop)+ if _is_or(expr):+ return self._source(expr.child_by_field_name("left"), loop, probe, hop=hop)+ if expr.type in ("identifier", "attribute"):+ return self._named_source(expr, loop, probe, hop=hop)+ if expr.type == "subscript" and self._slice_magnitude(expr) == "bounded":+ # A fixed-width slice bounds the trip count, not what is walked.+ walked = self._source(expr.child_by_field_name("value"), loop, probe, hop=hop)+ return walked[0], "bounded"+ if expr.type == "call":+ return self._call_source(expr, loop, probe, hop=hop)+ return "unknown", None++ def _named_source(self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:+ if lex.is_config_name(_text(expr)):+ # A registry grows with what users register: the name alone bounds nothing.+ return "config_table", None+ if expr.type == "identifier":+ return self._name_source(expr, loop, probe, hop=hop)+ return self._attribute_source(expr, loop, probe, hop=hop)++ def _attribute_source(self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:+ """``result.rows`` walks ``result``; ``self._TABLES`` bound to a literal is a constant."""+ attr = expr.child_by_field_name("attribute")+ name = (attr.text if attr is not None else b"") or b""+ if name.decode("utf-8", "replace") in _PY_RESULT_PROJECTIONS:+ return self._source(expr.child_by_field_name("object"), loop, probe, hop=hop)+ if self._literal_constant(expr, name):+ return "constant", "bounded"+ return "unknown", None++ def _name_source(self, name: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:+ rhs = self.reaching_value(loop, name.text or b"") if hop else None+ if rhs is not None:+ return self._source(rhs, loop, probe, hop=False)+ fn = loop.parent+ while fn is not None and fn.type != "function_definition":+ fn = fn.parent+ params = fn.child_by_field_name("parameters") if fn is not None else None+ if params is not None and _text(name) in self._param_names(params):+ return "parameter", None+ return "unknown", None++ def _call_source(self, call: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:+ method, fn, receiver, args = self._call_bits(call)+ if lex.is_config_name(self._dotted_path(fn) or "", callee=True):+ return "config_table", None+ if method == "range" and receiver is None:+ return self._range_source(call, args, loop, probe, hop=hop)+ walked = self._walked_arg_or_receiver(method, receiver, args)+ if walked is not None:+ return self._source(walked, loop, probe, hop=hop)+ if method in _PY_LISTINGS or method in _PY_STREAMS or (method == "open" and not receiver):+ return self._file_source(call, method, loop)+ if probe(call) in ("db", "network"):+ return self._query(call, loop)+ if receiver is not None and method in _PY_RESULT_PROJECTIONS:+ return self._source(receiver, loop, probe, hop=hop)+ return "unknown", None++ def _call_bits(self, call: Node) -> tuple[str, Node | None, Node | None, list[Node]]:+ """``(method, callee, receiver, positional args)`` of a call."""+ fn = call.child_by_field_name("function")+ receiver = fn.child_by_field_name("object") if fn is not None and fn.type == "attribute" else None+ arg_list = call.child_by_field_name("arguments")+ args = [c for c in arg_list.children if c.is_named] if arg_list is not None else []+ return self.callee_method_name(call) or "", fn, receiver, args++ @staticmethod+ def _walked_arg_or_receiver(+ method: str, receiver: Node | None, args: list[Node]+ ) -> Node | None:+ """``sorted(xs)`` / ``enumerate(xs)`` walk ``xs``; ``d.items()`` walks ``d``."""+ if method in _PY_PASS_THROUGH and args and args[0].type != "keyword_argument":+ return args[0]+ if receiver is not None and method in ("items", "values", "keys"):+ return receiver+ return None++ def _file_source(self, call: Node, method: str, loop: Node) -> Source:+ """A directory listing (grows when it walks the repository tree) or a stream."""+ if method in _PY_LISTINGS:+ return "listing", "grows_with_data" if self._walks_repo(call, method, loop) else None+ return "stream", None++ def _range_source(+ self, call: Node, args: list[Node], loop: Node, probe: SinkProbe, *, hop: bool+ ) -> Source:+ """``range(3)`` is a constant; ``range(0, len(ids), N)`` walks ``ids``."""+ if self._range_magnitude(call, loop, probe, hop=hop) == "bounded":+ return "constant", "bounded"+ stop = args[1] if len(args) > 1 else (args[0] if args else None)+ if stop is not None and stop.type == "call" and self.callee_method_name(stop) == "len":+ inner = [c for c in stop.child_by_field_name("arguments").children if c.is_named]+ if len(inner) == 1:+ return self._source(inner[0], loop, probe, hop=hop)+ return "unknown", None++ def reads_config_table(self, call: Node, nodes: list[Node]) -> bool:+ """Every SQL text and selected model names a registry or configuration table."""+ walked = [n for part in nodes for n in self._walk(part)]+ sql = [+ n.text or b""+ for n in walked+ if n.type == "string" and _PY_SQL_LITERAL_RE.match(_text(n))+ ]+ models = self._query_models(walked)+ return bool(sql or models) and (+ all(lex.is_config_sql(text) for text in sql)+ and all(lex.is_config_model(model) for model in models)+ )++ def _query_models(self, walked: list[Node]) -> list[str]:+ return [+ model+ for n in walked+ if n.type == "call" and self.callee_method_name(n) in ("select", "query")+ for model in self._selected_models(n)+ ]++ def _selected_models(self, select: Node) -> list[str]:+ """``select(Repository)`` / ``query(Repository.id, Repository.path)`` -> the model+ names; ``func.count()`` and other expressions name none."""+ args = select.child_by_field_name("arguments")+ models: list[str] = []+ for arg in (c for c in args.children if c.is_named) if args is not None else ():+ path = self._dotted_path(arg) or ""+ head = next((seg for seg in path.split(".") if _PY_MODEL_NAME_RE.match(seg)), None)+ if head is not None:+ models.append(head)+ return models++ def _literal_constant(self, expr: Node, name: bytes) -> bool:+ """``self._TABLES`` where the file binds ``_TABLES`` once, to a literal collection."""+ text = name.decode("utf-8", "replace")+ if len(text.strip("_")) < 2 or not text.isupper():+ return False+ root = expr+ while root.parent is not None:+ root = root.parent+ rights = [+ n.child_by_field_name("right")+ for n in self._walk(root)+ if n.type == "assignment"+ and (left := n.child_by_field_name("left")) is not None+ and left.type == "identifier"+ and left.text == name+ ]+ return (+ len(rights) == 1+ and rights[0] is not None+ and rights[0].type in _PY_CONSTANT_RHS+ and all(_scalar_literal(item) for item in rights[0].named_children)+ )++ def _magnitude(+ self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool+ ) -> LoopMagnitude | None:+ """Grows (a query result or a directory listing), bounded, or ``None``.++ *hop* allows one step through the name's last assignment before the+ loop; a parameter has none and stays unknown on purpose.+ """+ if expr is None:+ return None+ if expr.type in ("await", "parenthesized_expression"):+ inner = next((c for c in expr.children if c.is_named), None)+ return self._magnitude(inner, loop, probe, hop=hop)+ if expr.type == "identifier" and hop and expr.text:+ rhs = self.reaching_value(loop, expr.text)+ return self._magnitude(rhs, loop, probe, hop=False)+ if expr.type == "subscript":+ return self._slice_magnitude(expr)+ if _is_or(expr):+ # ``(await q.execute()).data or []``: the fallback is empty, the read is not.+ left = expr.child_by_field_name("left")+ return self._grows(self._magnitude(left, loop, probe, hop=hop))+ if expr.type == "attribute":+ return self._attribute_magnitude(expr, loop, probe, hop=hop)+ return self._call_magnitude(expr, loop, probe, hop=hop) if expr.type == "call" else None++ def _attribute_magnitude(+ self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool+ ) -> LoopMagnitude | None:+ attr = expr.child_by_field_name("attribute")+ if attr is None or (attr.text or b"").decode() not in _PY_RESULT_PROJECTIONS:+ return None+ return self._grows(self._magnitude(expr.child_by_field_name("object"), loop, probe, hop=hop))++ def _call_magnitude(+ self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool+ ) -> LoopMagnitude | None:+ method = self.callee_method_name(expr) or ""+ member = self.callee_is_attribute(expr)+ if method == "range" and not member:+ return self._range_magnitude(expr, loop, probe, hop=hop)+ if probe(expr) in ("db", "network"):+ # A read capped in the query is as large as its cap; a statement or result+ # built before the loop (``stmt = ....limit(n)``) is read one assignment back.+ names = [n.text for n in self._walk(expr) if n.type == "identifier"]+ parts = [expr, *filter(None, (self.reaching_value(loop, name) for name in names))]+ capped = any(self.is_cap(n, loop) for part in parts for n in self._walk(part))+ return None if capped else "grows_with_data"+ if member and method in _PY_RESULT_PROJECTIONS:+ receiver = expr.child_by_field_name("function").child_by_field_name("object")+ return self._grows(self._magnitude(receiver, loop, probe, hop=hop))+ return None++ def is_cap(self, node: Node, loop: Node) -> bool:+ """``.limit(n)`` and peers, a SQL string saying ``LIMIT n``, or a filter to a+ set of keys (:meth:`read_key_column`), which is as large as the keys are."""+ if node.type == "string":+ return bool(_PY_SQL_LIMIT_RE.search(_PY_SQL_COMMENT_RE.sub(b"", node.text or b"")))+ method = self.callee_method_name(node) if node.type == "call" else None+ if method in _PY_READ_CAPS:+ return True+ return node.type == "call" and self.read_key_column(node, loop) is not None++ def read_key_column(self, call: Node, anchor: Node) -> str | None:+ """``Model.col.in_(keys)`` and PostgREST ``.in_("col", keys)`` -> ``col``;+ ``filter_by(id=k)`` and ``.eq("id", k)`` -> ``id``. Keys a query yields+ (``.in_(select(...))``, or a name bound to one) grow with that table."""+ method = self.callee_method_name(call)+ arg_list = call.child_by_field_name("arguments")+ args = arg_list.named_children if arg_list is not None else []+ if method == "filter_by":+ named = {_field_text(a, "name") for a in args if a.type == "keyword_argument"}+ return "id" if "id" in named else None+ if method == "eq":+ return "id" if len(args) == 2 and _string_value(args[0]) == "id" else None+ found = _in_filter(call, args) if method == "in_" else None+ if found is None or not self._handed_in(found[1], anchor):+ return None+ return found[0]++ def _handed_in(self, keys: Node, anchor: Node) -> bool:+ """*keys* holds values given to the code, not ones a query yields or a fixed+ set of states (``in_(("a", "b"))``, ``in_(SOURCES)``) whose rows grow."""+ values = [+ self.reaching_value(anchor, ref.text or b"") if ref.type == "identifier" else None+ for ref in _value_refs(keys)+ ]+ if all(_fixed_set(value) for value in values):+ return False+ parts = [keys, *filter(None, values)]+ return not any(+ n.type == "call" and self.callee_method_name(n) in _PY_KEY_READS+ for part in parts+ for n in self._walk(part)+ )++ def _range_magnitude(+ self, call: Node, loop: Node, probe: SinkProbe, *, hop: bool+ ) -> LoopMagnitude | None:+ """``range(len(rows))`` grows with ``rows``; ``range(3)`` and ``range(MAX)``+ with ``MAX = 3`` bound once in the module are bounded."""+ args = [c for c in call.child_by_field_name("arguments").children if c.is_named]+ if not args:+ return None+ stop = args[1] if len(args) > 1 else args[0]+ if stop.type == "call" and self.callee_method_name(stop) == "len":+ inner = [c for c in stop.child_by_field_name("arguments").children if c.is_named]+ if len(inner) == 1:+ return self._grows(self._magnitude(inner[0], loop, probe, hop=hop))+ return None+ # A name alone (``MAX_RETRIES``, ``self.max_attempts``) proves nothing.+ return "bounded" if stop.type == "integer" or _module_int_constant(stop) else None++ def _slice_magnitude(self, subscript: Node) -> LoopMagnitude | None:+ """``xs[:5]`` / ``xs[:LIMIT]`` / ``xs[i:i + 4]``: a constant-width slice."""+ bounds = _slice_bounds(subscript)+ if bounds is None:+ return None+ start, stop = bounds+ if start is None:+ return "bounded" if stop.type == "integer" or _module_int_constant(stop) else None+ return "bounded" if self._offset_by_int(start, stop) else None++ def _offset_by_int(self, start: Node, stop: Node) -> bool:+ """*stop* is ``start + <integer>``."""+ if stop.type != "binary_operator":+ return False+ left, right = stop.child_by_field_name("left"), stop.child_by_field_name("right")+ same_start = left is not None and self._dotted_path(left) == self._dotted_path(start)+ return same_start and right is not None and right.type == "integer"