Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(health): split the Extract Method slicer and the Python perf dialect into focused modules (#4025)

Raghav Chamadiyaa10d3d62734e
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice.py
index c6eb59363..4102e737b 100644
--- a/packages/core/src/repowise/core/analysis/health/dataflow/slice.py
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice.py
@@ -1,9 +1,8 @@
"""Extract Method slicing: find safe, single-exit spans with IN/OUT inference.
-The first user-facing consumer of the dataflow layer. Given a function's
-analysis (CFG + def/use from D2), it finds contiguous statement spans that can
-be lifted into a helper method without changing behaviour, and infers the
-helper's signature:
+Given a function's analysis (CFG + def/use), it finds contiguous statement
+spans that can be lifted into a helper method without changing behaviour, and
+infers the helper's signature:
- **IN (parameters)** -- variables the span *reads* whose value is produced
before the span (defined-before, used-inside).
@@ -19,13 +18,13 @@ matter without being nearly the whole body, and has at most one return and a
small parameter list. Everything else is suppressed -- ten great extractions,
not two hundred maybes.
-Line-based liveness over D2's def/use occurrences realises the IN inference,
-reaching definitions over the CFG the OUT inference, and the CFG/jump scan the
-single-exit predicate. The jump and nested-scope node kinds come from the
-language's ``LanguageNodeMap`` (the same source the CFG builder uses), so every
-full-tier language whose map populates them is served by this one slicer.
+Node kinds come from the language's ``LanguageNodeMap``, so one slicer serves
+every full-tier language. The helpers live in the ``slice_*`` modules and are
+re-exported here; patch a name where it is looked up.
"""
+# ruff: noqa: F401
+
from __future__ import annotations
from bisect import bisect_left, bisect_right
@@ -39,16 +38,88 @@ from ..complexity.nloc import _code_line_numbers
from .dialects.base import SUPER, mentions_receiver
from .raises import exception_flow_refusal
from .reaching import definitions_at, observed_definitions
+from .slice_liveness import (
+ _bindings_escape,
+ _declaration_escapes,
+ _declaration_lines,
+ _declared_before_read,
+ _hoisted_bindings,
+ _import_reads,
+ _infer_in_out,
+ _needed_after,
+ _outer_write_escapes,
+ _reads_observing,
+ _statements_by_line,
+ _valueless_lines,
+ _var_lines,
+)
+from .slice_nodes import (
+ _BINDER_FIELDS,
+ _awaits,
+ _binder_names,
+ _descend,
+ _exit_macros,
+ _function_lines,
+ _holds_a_named_nested_function,
+ _identifiers,
+ _loop_header_names,
+ _receiver_root,
+ _unwrap_container,
+ all_blocks,
+)
+from .slice_receiver import (
+ _FUNCTION_KINDS,
+ _INDEX_KINDS,
+ _UNPACK_KINDS,
+ _UNREAD_TARGET,
+ FunctionFacts,
+ _collect_assigned,
+ _ends_in_exit,
+ _free_write_lines,
+ _function_kinds,
+ _is_receiver,
+ _receiver_facts,
+ _receiver_field,
+ _receiver_kinds,
+ _receiver_ref,
+ _ReceiverSink,
+ _whole_receiver_facts,
+ body_tally,
+ function_facts,
+)
+from .slice_state import (
+ _closure_state,
+ _closure_state_crosses,
+ _count_in,
+ _declared_above,
+ _declares_in,
+ _loop_carry_free,
+ _rebinding_defs_dropped,
+ _SharedState,
+ _span_binds,
+ _span_only_rebinds,
+)
+from .slice_writes import (
+ _branch_assigns,
+ _conditional_assigns,
+ _is_conditional,
+ _outs_definitely_assigned,
+ _stmt_assigns,
+ _stmts_assign,
+ _unconditional_write,
+ _writes_var,
+)
if TYPE_CHECKING:
+ from collections.abc import Iterator
+
from tree_sitter import Node
from ..complexity.languages import LanguageNodeMap
from .analyze import FunctionAnalysis
- from .cfg import CFG
- from .defuse import FunctionDefUse
from .dialects.base import Receiver
+
# Gates (precision-first; tuned to suppress trivial or unwieldy extractions).
_MIN_STMTS = 2 # at least two statements
# The extracted helper is substantial: code lines, the walker's NLOC rule, so a
@@ -56,15 +127,12 @@ _MIN_STMTS = 2 # at least two statements
_MIN_SLICE_NLOC = 5
_MIN_CCN_REMOVED = 1 # extraction must remove a real decision point
_MAX_PARAMS = 5 # too many ins => the span is not cohesive
-_MAX_RETURNS = 1 # a single clean return (v1); multi-output is future work
+_MAX_RETURNS = 1 # a single clean return
# Backstop against a pathological function producing too many sub-ranges.
_MAX_CANDIDATES = 4000
# A span holding this share of the body's code lines or more is the whole
-# function under another name, not a split: what stays behind is a guard, a
-# wrapper (``try`` / ``with``) or the final ``return``. Measured on 301 plans
-# from seven repos (Python, TS, Go, Java, Rust): 52 sat at or above 0.75, and
-# nearly all of those at 0.8 or more left only such a shell. The span offered
-# instead may be the same block minus a statement, just under the cut.
+# function under another name: what stays behind is only a guard, a wrapper or
+# the final ``return``.
_MAX_BODY_SHARE = 0.75
@@ -157,149 +225,229 @@ def find_extractions(
"""Return safe Extract Method candidates for *analysis*, best first.
Best is most complexity removed, then largest span, then fewest parameters,
- then earliest -- a deterministic order. Empty when the function has no AST
- node retained or no span clears the extractability gates. *receiver* is
- the function's instance (``DefUseDialect.receiver``), None when unknown.
+ then earliest. *receiver* is the function's instance, None when unknown.
*prefixes*, when given, receives each walked block's :class:`_Prefix` by
- block id, so a staged plan reads them instead of walking again.
- *refused*, when given, receives why a span that cleared every other gate
- was dropped because lifting it would change exception flow
- (:func:`raises.exception_flow_refusal`).
+ block id; *refused* receives why a span that cleared every other gate would
+ change exception flow (:func:`raises.exception_flow_refusal`).
"""
fn_node = analysis.fn_node
- if fn_node is None:
- return []
- # A subtree tree-sitter could not parse is not a function whose statements
- # we understand, so no span in it is safe to lift. This fires on macro-heavy
- # C/C++ headers, where an unterminated function-like macro
- # (``ABSL_NAMESPACE_BEGIN``) makes the parser emit one bogus
- # ``function_definition`` spanning a whole class or namespace — proposing to
- # extract "statements" from that is a wrong suggestion, not a weak one.
- # Language-agnostic and strictly subtractive: a clean parse is unaffected.
- if fn_node.has_error:
+ # A subtree tree-sitter could not parse (a macro-heavy C/C++ header emitting
+ # one bogus function over a whole namespace) has no statements we understand.
+ if fn_node is None or fn_node.has_error:
return []
body = fn_node.child_by_field_name("body")
if body is None:
return []
- # The statement container of the body (Go nests it in a ``statement_list``
- # inside the ``block``); spans covering it whole are not extractions.
- body_container = _unwrap_container(body, lmap.block_kinds)
- lines = _function_lines(fn_node)
- max_slice_nloc = _MAX_BODY_SHARE * _stmts_nloc(body_container.named_children, lines)
-
- def_lines, use_lines = _var_lines(analysis.def_use)
- import_reads = _import_reads(analysis)
- declared_first = _declared_before_read(analysis.def_use)
- valueless = _valueless_lines(analysis.def_use)
- # Built on the first span that clears the cheap gates.
- reads: dict[str, list[tuple[int, frozenset[int]]]] | None = None
- hoisted = _hoisted_bindings(def_lines, use_lines)
- decl_lines = _declaration_lines(analysis.def_use)
- shared = _closure_state(analysis.def_use, def_lines, use_lines)
- outer = analysis.def_use.outer
- carry_defs = _rebinding_defs_dropped(analysis, def_lines, lmap)
- scan = _scan_for(lmap, receiver, fn_node)
- scope_kinds = scan.scopes
- free_writes = _free_write_lines(analysis.def_use) if receiver and receiver.implicit else []
- # Expression-oriented grammars (nonempty ``statement_wrapper_kinds``): a
- # block's last child that is not a statement is its tail expression -- the
- # block's implicit value. A span ending on one would silently drop that
- # value when lifted into a helper, so such spans are refused outright.
- tail_stmt_kinds = (
- lmap.statement_wrapper_kinds | lmap.local_decl_kinds
- if lmap.statement_wrapper_kinds
- else None
- )
-
+ slicer = _FunctionSlicer(analysis, lmap, receiver, fn_node, body)
out: list[Extraction] = []
- evaluated = 0
- for block, loop in all_blocks(fn_node, lmap.block_kinds, scope_kinds, lmap.loop_kinds):
+ for block, loop in all_blocks(fn_node, lmap.block_kinds, slicer.scan.scopes, lmap.loop_kinds):
+ if not slicer.block_extractions(block, loop, prefixes, refused, out):
+ break
+ return _sorted(out)
+
+
+class _Block(NamedTuple):
+ """One statement container: its statements and per-block facts."""
+
+ node: Node
+ loop: Node | None
+ stmts: list[Node]
+ pre: _Prefix
+ resources: list[int]
+
+
+class _Span(NamedTuple):
+ """Statements ``i..j`` of a block, spanning lines ``start..end``."""
+
+ i: int
+ j: int
+ nodes: list[Node]
+ start: int
+ end: int
+
+
+class _FunctionSlicer:
+ """The per-function facts every candidate span is checked against, built once."""
+
+ def __init__(
+ self,
+ analysis: FunctionAnalysis,
+ lmap: LanguageNodeMap,
+ receiver: Receiver | None,
+ fn_node: Node,
+ body: Node,
+ ) -> None:
+ self.analysis = analysis
+ self.lmap = lmap
+ self.receiver = receiver
+ self.fn_node = fn_node
+ # Go nests the statements in a ``statement_list``; spans covering it whole are not extractions.
+ self.body_container = _unwrap_container(body, lmap.block_kinds)
+ self.lines = _function_lines(fn_node)
+ self.max_slice_nloc = _MAX_BODY_SHARE * _stmts_nloc(
+ self.body_container.named_children, self.lines
+ )
+ def_use = analysis.def_use
+ self.def_lines, self.use_lines = _var_lines(def_use)
+ self.import_reads = _import_reads(analysis)
+ self.declared_first = _declared_before_read(def_use)
+ self.valueless = _valueless_lines(def_use)
+ # Built on the first span that clears the cheap gates.
+ self.reads: dict[str, list[tuple[int, frozenset[int]]]] | None = None
+ self.hoisted = _hoisted_bindings(self.def_lines, self.use_lines)
+ self.decl_lines = _declaration_lines(def_use)
+ self.shared = _closure_state(def_use, self.def_lines, self.use_lines)
+ self.outer = def_use.outer
+ self.carry_defs = _rebinding_defs_dropped(analysis, self.def_lines, lmap)
+ self.scan = _scan_for(lmap, receiver, fn_node)
+ self.free_writes = _free_write_lines(def_use) if receiver and receiver.implicit else []
+ # Expression-oriented grammars: a block's last non-statement child is its
+ # value, and a span ending on it would drop that value.
+ self.tail_stmt_kinds = (
+ lmap.statement_wrapper_kinds | lmap.local_decl_kinds
+ if lmap.statement_wrapper_kinds
+ else None
+ )
+ self.evaluated = 0
+ self.exhausted = False
+
+ def block_extractions(
+ self,
+ block: Node,
+ loop: Node | None,
+ prefixes: dict[int, _Prefix] | None,
+ refused: set[str] | None,
+ out: list[Extraction],
+ ) -> bool:
+ """Append *block*'s candidates to *out*; False once the candidate budget is spent."""
+ lmap = self.lmap
stmts = block.named_children
- # A match/switch body holds the arms, not statements: a span of them is
- # neither a function body nor a valid call site. Each arm's own body is
- # still a block of its own.
+ # A match/switch body holds arms, not statements; each arm's body is a block of its own.
if any(stmt.type in lmap.case_kinds for stmt in stmts):
- continue
+ return True
n = len(stmts)
resources = _scoped_resource_indexes(stmts, lmap)
- is_body = block.id == body_container.id
- keeps_tail = tail_stmt_kinds is not None and _tail_is_block_value(
- stmts, tail_stmt_kinds, fn_node, lmap
+ keeps_tail = self.tail_stmt_kinds is not None and _tail_is_block_value(
+ stmts, self.tail_stmt_kinds, self.fn_node, lmap
)
- if n >= _MIN_STMTS:
- pre = _block_prefix(stmts, scan, lines, lmap)
- if prefixes is not None:
- prefixes[block.id] = pre
+ spans = self._span_bounds(n, block.id == self.body_container.id, keeps_tail)
+ if n < _MIN_STMTS:
+ # No span is long enough, but each still counts against the budget.
+ for _ in spans:
+ pass
+ return not self.exhausted
+ pre = _block_prefix(stmts, self.scan, self.lines, lmap)
+ if prefixes is not None:
+ prefixes[block.id] = pre
+ blk = _Block(block, loop, stmts, pre, resources)
+ for i, j in spans:
+ decisions = pre.decisions[j + 1] - pre.decisions[i]
+ if pre.jumps[j + 1] > pre.jumps[i] or decisions < _MIN_CCN_REMOVED:
+ continue
+ slice_nloc = pre.code[j + 1] - pre.code[i]
+ if not _MIN_SLICE_NLOC <= slice_nloc < self.max_slice_nloc:
+ continue
+ found = self._extraction(blk, i, j, decisions, slice_nloc, refused)
+ if found is not None:
+ out.append(found)
+ return not self.exhausted
+
+ def _span_bounds(self, n: int, is_body: bool, keeps_tail: bool) -> Iterator[tuple[int, int]]:
+ """Ranges ``(i, j)`` long enough to lift, never the whole body or a block's
+ value tail. Every range counts against the candidate budget."""
for i in range(n):
for j in range(i, n):
- evaluated += 1
- if evaluated > _MAX_CANDIDATES:
- return _sorted(out)
+ self.evaluated += 1
+ if self.evaluated > _MAX_CANDIDATES:
+ self.exhausted = True
+ return
length = j - i + 1
- if length < _MIN_STMTS:
- continue
- # Never extract the whole function body (that is not a split).
- if is_body and length == n:
- continue
- if keeps_tail and j == n - 1:
- continue
- decisions = pre.decisions[j + 1] - pre.decisions[i]
- has_jump = pre.jumps[j + 1] > pre.jumps[i]
- if has_jump or decisions < _MIN_CCN_REMOVED:
- continue
- slice_nloc = pre.code[j + 1] - pre.code[i]
- if not _MIN_SLICE_NLOC <= slice_nloc < max_slice_nloc:
- continue
- span = stmts[i : j + 1]
- s = span[0].start_point[0] + 1
- e = span[-1].end_point[0] + 1
- if any(import_line < s <= use_line <= e for import_line, use_line in import_reads):
- continue
- if reads is None:
- reads = _reads_observing(analysis, def_lines)
- params, returns = _infer_in_out(
- def_lines, use_lines, s, e, declared_first, reads=reads, valueless=valueless
- )
- if len(params) > _MAX_PARAMS or len(returns) > _MAX_RETURNS:
- continue
- if not _outs_definitely_assigned(span, returns, def_lines, lmap):
- continue
- if any(s <= first_def <= e and first_use < s for first_def, first_use in hoisted):
- continue
- if _bindings_escape(
- s, e, returns, decl_lines, def_lines, use_lines, declared_first, outer
- ):
- continue
- if _disposes_early(resources, i, j, pre.code):
- continue
- if pre.nested[j + 1] > pre.nested[i]:
- continue
- if shared is not None and _closure_state_crosses(shared, span, s, e, block, lmap):
+ if length < _MIN_STMTS or (is_body and length == n) or (keeps_tail and j == n - 1):
continue
- if loop is not None and not _loop_carry_free(
- span, loop, s, e, carry_defs, use_lines, lmap
- ):
- continue
- reason = exception_flow_refusal(analysis.cfg, s, e, reads, def_lines)
- if reason is not None and refused is not None:
- refused.add(reason)
- if reason is not None:
- continue
- uses, assigns = _receiver_facts(receiver, pre, i, j, _count_in(free_writes, s, e))
- out.append(
- Extraction(
- start_line=s,
- end_line=e,
- params=params,
- returns=returns,
- slice_nloc=slice_nloc,
- ccn_removed=decisions,
- needs_async=pre.awaits[j + 1] > pre.awaits[i],
- uses_receiver=uses,
- receiver_assigns=assigns,
- )
- )
- return _sorted(out)
+ yield i, j
+
+ def _extraction(
+ self,
+ blk: _Block,
+ i: int,
+ j: int,
+ decisions: int,
+ slice_nloc: int,
+ refused: set[str] | None,
+ ) -> Extraction | None:
+ nodes = blk.stmts[i : j + 1]
+ span = _Span(i, j, nodes, nodes[0].start_point[0] + 1, nodes[-1].end_point[0] + 1)
+ interface = self._interface(span)
+ if interface is None or not self._self_contained(blk, span, interface[1]):
+ return None
+ s, e = span.start, span.end
+ reason = exception_flow_refusal(self.analysis.cfg, s, e, self.reads, self.def_lines)
+ if reason is not None:
+ if refused is not None:
+ refused.add(reason)
+ return None
+ params, returns = interface
+ pre = blk.pre
+ free_writes = _count_in(self.free_writes, s, e)
+ uses, assigns = _receiver_facts(self.receiver, pre, i, j, free_writes)
+ return Extraction(
+ start_line=s,
+ end_line=e,
+ params=params,
+ returns=returns,
+ slice_nloc=slice_nloc,
+ ccn_removed=decisions,
+ needs_async=pre.awaits[j + 1] > pre.awaits[i],
+ uses_receiver=uses,
+ receiver_assigns=assigns,
+ )
+
+ def _interface(self, span: _Span) -> tuple[tuple[str, ...], tuple[str, ...]] | None:
+ """The span's ``(params, returns)``, or None when they are too many or an
+ OUT is not written on every path."""
+ s, e = span.start, span.end
+ if any(import_line < s <= use_line <= e for import_line, use_line in self.import_reads):
+ return None
+ if self.reads is None:
+ self.reads = _reads_observing(self.analysis, self.def_lines)
+ params, returns = _infer_in_out(
+ self.def_lines,
+ self.use_lines,
+ s,
+ e,
+ self.declared_first,
+ reads=self.reads,
+ valueless=self.valueless,
+ )
+ if len(params) > _MAX_PARAMS or len(returns) > _MAX_RETURNS:
+ return None
+ if not _outs_definitely_assigned(span.nodes, returns, self.def_lines, self.lmap):
+ return None
+ return params, returns
+
+ def _self_contained(self, blk: _Block, span: _Span, returns: tuple[str, ...]) -> bool:
+ """No binding, resource, nested function, closure state or loop-carried
+ value ties the span to code it does not hold."""
+ s, e = span.start, span.end
+ if any(s <= first_def <= e and first_use < s for first_def, first_use in self.hoisted):
+ return False
+ if _bindings_escape(
+ s, e, returns, self.decl_lines, self.def_lines, self.use_lines,
+ self.declared_first, self.outer,
+ ):
+ return False
+ pre = blk.pre
+ if _disposes_early(blk.resources, span.i, span.j, pre.code):
+ return False
+ if pre.nested[span.j + 1] > pre.nested[span.i]:
+ return False
+ if self.shared is not None and _closure_state_crosses(
+ self.shared, span.nodes, s, e, blk.node, self.lmap
+ ):
+ return False
+ return blk.loop is None or _loop_carry_free(
+ span.nodes, blk.loop, s, e, self.carry_defs, self.use_lines, self.lmap
+ )
def _scoped_resource_indexes(stmts: list[Node], lmap: LanguageNodeMap) -> list[int]:
@@ -323,119 +471,6 @@ def _disposes_early(resources: list[int], i: int, j: int, code: list[int]) -> bo
return code[-1] > code[j + 1] and any(i <= k <= j for k in resources)
-@dataclass(frozen=True)
-class FunctionFacts:
- """What a whole function does, counted on the complexity walk's visits.
-
- ``awaits``: it suspends outside any nested scope. ``is_generator``: it
- yields. ``uses_receiver`` / ``receiver_assigns`` as on :class:`Extraction`,
- for the whole body; ``receiver_assigns`` is None where an implicit receiver
- (Java, C++) can write fields by bare name, which only def/use can tell.
- ``early_exits``: returns and raises other than a final one.
- """
-
- awaits: bool
- is_generator: bool
- uses_receiver: bool | None
- receiver_assigns: tuple[str, ...] | None
- early_exits: int
-
-
-class _ReceiverSink:
- """``BodyTally.on_receiver``: whether the body names its receiver, and
- which of its fields it assigns."""
-
- __slots__ = ("assigns", "receiver", "uses")
-
- def __init__(self, receiver: Receiver) -> None:
- self.receiver = receiver
- self.uses = False
- self.assigns: set[str] = set()
-
- def __call__(self, node: Node) -> None:
- self.uses = _receiver_ref(node, self.receiver, self.assigns) or self.uses
-
-
-def body_tally(fn_node: Node, lmap: LanguageNodeMap, receiver: Receiver | None) -> BodyTally:
- """An empty tally for *fn_node*, for the complexity walk to fill. The
- receiver is watched only when the function's text names it."""
- watched = bool(
- receiver is not None and receiver.names and mentions_receiver(fn_node, receiver.names)
- )
- sink = _ReceiverSink(receiver) if watched and receiver is not None else None
- return BodyTally(_function_kinds(lmap, sink), sink)
-
-
-# Built once per language map and receiver shape: the walk asks per function.
-_FUNCTION_KINDS: dict[tuple[int, frozenset[str]], FactKinds] = {}
-
-
-def _function_kinds(lmap: LanguageNodeMap, sink: _ReceiverSink | None) -> FactKinds:
- receiver = _receiver_kinds(sink.receiver) if sink else frozenset()
- key = (id(lmap), receiver)
- kinds = _FUNCTION_KINDS.get(key)
- if kinds is None:
- kinds = _FUNCTION_KINDS[key] = FactKinds(
- exits=lmap.return_kinds | lmap.raise_kinds,
- exit_macros=_exit_macros(lmap),
- yields=lmap.yield_kinds,
- awaits=lmap.await_kinds,
- await_scopes=lmap.await_scope_kinds,
- lambdas=lmap.lambda_kinds,
- receiver=receiver,
- )
- return kinds
-
-
-def _receiver_kinds(receiver: Receiver) -> frozenset[str]:
- """What :func:`_receiver_ref` can answer for: a write, or a receiver leaf."""
- return receiver.write_kinds | receiver.names | {SUPER, "identifier"}
-
-
-def function_facts(
- fn_node: Node, lmap: LanguageNodeMap, receiver: Receiver | None, tally: BodyTally
-) -> FunctionFacts:
- """:class:`FunctionFacts` from a filled :func:`body_tally`."""
- body = fn_node.child_by_field_name("body") or fn_node
- sink = tally.on_receiver if isinstance(tally.on_receiver, _ReceiverSink) else None
- uses, assigns = _whole_receiver_facts(
- receiver, sink.uses if sink else False, frozenset(sink.assigns) if sink else frozenset()
- )
- return FunctionFacts(
- awaits=tally.awaits,
- is_generator=tally.yields > 0,
- uses_receiver=uses,
- receiver_assigns=assigns,
- early_exits=max(0, tally.exits - _ends_in_exit(body, lmap, tally.kinds)),
- )
-
-
-def _ends_in_exit(body: Node, lmap: LanguageNodeMap, kinds: FactKinds) -> bool:
- """Whether a block body's last statement leaves the function, so that exit
- is not an early one. The statement is the exit (Python ``return x``, TS
- ``throw e;``) or wraps it as its first named child (Rust ``return x;`` and
- ``bail!(..);`` are expression statements). An expression body has no
- statements, so its exits are early by definition and none are subtracted."""
- if body.type not in lmap.block_kinds or not body.named_children:
- return False
- tail = body.named_children[-1]
- if is_exit(tail, kinds):
- return True
- return bool(tail.named_children) and is_exit(tail.named_children[0], kinds)
-
-
-def _whole_receiver_facts(
- receiver: Receiver | None, uses: bool, assigned: frozenset[str]
-) -> tuple[bool | None, tuple[str, ...] | None]:
- if receiver is None:
- return None, None
- if not receiver.names:
- return False, ()
- if receiver.implicit:
- return (True if uses else None), None
- return uses, None if _UNREAD_TARGET in assigned else tuple(sorted(assigned))
-
-
def _scan_for(lmap: LanguageNodeMap, receiver: Receiver | None, fn_node: Node) -> _Scan:
"""The walk's kinds for *fn_node*. The receiver is looked for only when
its name (or ``super``) appears in the function's text at all: most
@@ -486,34 +521,6 @@ def _block_prefix(
return pre
-def _free_write_lines(def_use: FunctionDefUse) -> list[int]:
- """Sorted lines writing a name the function neither declares nor takes
- as a parameter: in Java or C++ that is a field (or a C++ global) written
- without ``this``, so the span's receiver writes are not all known."""
- params = {p.name for p in def_use.params}
- declared = {d.var for d in def_use.definitions if d.declares or d.declared_at is not None}
- return sorted(
- d.line for d in def_use.definitions if d.var not in params and d.var not in declared
- )
-
-
-def _receiver_facts(
- receiver: Receiver | None, pre: _Prefix, i: int, j: int, free_writes: int
-) -> tuple[bool | None, tuple[str, ...] | None]:
- """``uses_receiver`` / ``receiver_assigns`` for the span over statements
- ``i..j`` (see :class:`Extraction`)."""
- if receiver is None:
- return None, None
- if not receiver.names:
- return False, ()
- named = pre.receiver[j + 1] > pre.receiver[i]
- found = frozenset().union(*pre.assigns[i : j + 1])
- fields = None if _UNREAD_TARGET in found else tuple(sorted(found))
- if receiver.implicit:
- return (True if named else None), (None if free_writes else fields)
- return named, fields
-
-
def _tail_is_block_value(
stmts: list[Node], tail_stmt_kinds: frozenset[str], fn_node: Node, lmap: LanguageNodeMap
) -> bool:
@@ -566,16 +573,6 @@ def _value_hop(node: Node, parent: Node, lmap: LanguageNodeMap) -> bool | None:
return True # a let initializer, an argument, an operand
-def _function_lines(fn_node: Node) -> list[str]:
- """Source rows indexed by absolute row, rebuilt from the function's own text.
-
- Lets the walker's NLOC rule (``_code_line_numbers``) run here without the
- file's bytes; rows above the function are never read, so they stay empty.
- """
- text = (fn_node.text or b"").decode("utf-8", errors="replace")
- return [""] * fn_node.start_point[0] + text.splitlines()
-
-
def _stmts_nloc(stmts: list[Node], lines: list[str]) -> int:
"""Code lines of *stmts* by the walker's NLOC rule, summed per statement."""
return sum(len(_code_line_numbers(st, lines, drop_docstrings=True)) for st in stmts)
@@ -588,862 +585,6 @@ def _sorted(candidates: list[Extraction]) -> list[Extraction]:
)
-def _var_lines(def_use: FunctionDefUse) -> tuple[dict[str, list[int]], dict[str, list[int]]]:
- """Per-variable sorted def lines and use lines from D2's facts.
-
- Parameter definitions are included (seeded at the signature line), so a
- parameter naturally counts as "defined before" any body span. Reads inside
- nested closures (``def_use.captured.reads``) count as uses at their own line.
- An import is not a def here: a helper re-imports a name rather than take
- or return it, and :func:`_declaration_escapes` refuses a span holding an
- import that later code still reads.
- """
- def_lines: dict[str, list[int]] = defaultdict(list)
- use_lines: dict[str, list[int]] = defaultdict(list)
- imported: set[str] = set()
- for d in def_use.definitions:
- if d.imports:
- imported.add(d.var)
- else:
- def_lines[d.var].append(d.line)
- for bdu in def_use.blocks.values():
- for u in bdu.uses:
- # A may-def's paired use is bookkeeping, not a read: counted, a
- # binder declared inside a ``match`` arm became its own parameter.
- if not u.echo:
- use_lines[u.name].append(u.line)
- # A closure's read counts where the closure is written: lifting the code
- # around it moves the read with it. Only names this function binds matter.
- for u in def_use.captured.reads:
- if u.name in def_lines or u.name in imported:
- use_lines[u.name].append(u.line)
- for lines in def_lines.values():
- lines.sort()
- for lines in use_lines.values():
- lines.sort()
- return def_lines, use_lines
-
-
-def _import_reads(analysis: FunctionAnalysis) -> tuple[tuple[int, int], ...]:
- """Function-local import lines paired with the reads they reach."""
- imports = {
- (definition.var, definition.line)
- for definition in analysis.reaching.definitions
- if definition.imports
- }
- if not imports:
- return ()
- import_lines: dict[str, list[int]] = defaultdict(list)
- for name, line in sorted(imports):
- import_lines[name].append(line)
- return tuple(
- (definition_line, use_line)
- for name, reads in _reads_observing(analysis, import_lines).items()
- for use_line, seen in reads
- for definition_line in seen
- if (name, definition_line) in imports
- )
-
-
-def _declared_before_read(def_use: FunctionDefUse) -> dict[str, frozenset[int]]:
- """Per variable, the lines where a declaration of it comes before its first
- read on the same line.
-
- Lines alone cannot order a write and a read that share one. A C-style
- ``for (int i = 0; i < n; i++)`` declares ``i`` and then reads it, while
- ``total = total + a[i]`` reads ``total`` and then writes it. Only a
- declaration is ordered here, by where its declarator ends: a read past that
- point sees the new name, and a read inside the declaration's own
- initializer (Go's ``x := x + 1`` in an inner scope) still sees the outer
- one. A plain assignment keeps the line rule.
- """
- first_read: dict[tuple[str, int], int] = {}
- for bdu in def_use.blocks.values():
- for u in bdu.uses:
- if u.echo:
- continue
- key = (u.name, u.line)
- first_read[key] = min(u.column, first_read.get(key, u.column))
- declared: dict[str, set[int]] = defaultdict(set)
- for d in def_use.definitions:
- # No read on the line sorts before any declarator, so it never matches.
- if d.declared_at is not None and d.declared_at <= first_read.get((d.var, d.line), -1):
- declared[d.var].add(d.line)
- return {var: frozenset(lines) for var, lines in declared.items()}
-
-
-def _valueless_lines(def_use: FunctionDefUse) -> dict[str, frozenset[int]]:
- """Per variable, the lines whose only definitions of it are declarations
- that give it no value (``Definition.valueless``): Java's ``List<T> xs;``,
- TypeScript's ``let x: T;``. A line that also assigns the name
- (``int x; x = 1;``) is not one: what a read sees from it is the value."""
- bare: dict[str, set[int]] = defaultdict(set)
- valued: dict[str, set[int]] = defaultdict(set)
- for d in def_use.definitions:
- (bare if d.valueless else valued)[d.var].add(d.line)
- return {var: frozenset(at - valued[var]) for var, at in bare.items()}
-
-
-def _reads_observing(
- analysis: FunctionAnalysis, def_lines: dict[str, list[int]]
-) -> dict[str, list[tuple[int, frozenset[int]]]]:
- """Per variable, each read's line with the def lines it may observe.
-
- Reads take reaching definitions over the CFG, so a later write on a path
- that need not run (an ``else`` arm, a ``try`` body that may raise first)
- does not hide the span's value from the read. A closure's read observes
- what reaches the statement that creates the closure, or, where no recorded
- statement covers it, the nearest def above it: a read is never dropped,
- since a dropped read can only drop a return. Only names in *def_lines*
- count for closures, as in :func:`_var_lines`.
- """
- def_use, reaching = analysis.def_use, analysis.reaching
- defs = reaching.definitions
- reads: dict[str, list[tuple[int, frozenset[int]]]] = defaultdict(list)
- for use, seen in observed_definitions(def_use, reaching):
- if not use.echo:
- reads[use.name].append((use.line, frozenset(defs[i].line for i in seen)))
- statements: dict[int, tuple[int, int]] | None = None
- for u in def_use.captured.reads:
- dl = def_lines.get(u.name)
- if not dl:
- continue
- if statements is None:
- statements = _statements_by_line(analysis.cfg)
- at = statements.get(u.line)
- if at is not None:
- observed = frozenset(
- defs[i].line for i in definitions_at(def_use, reaching, at[0], u.name, at[1])
- )
- else:
- k = bisect_left(dl, u.line)
- observed = frozenset(dl[k - 1 : k])
- reads[u.name].append((u.line, observed))
- return reads
-
-
-def _statements_by_line(cfg: CFG) -> dict[int, tuple[int, int]]:
- """Per line, (block id, start line) of the innermost recorded statement
- covering it."""
- best: dict[int, tuple[int, int, int]] = {}
- for block in cfg.blocks:
- for st in block.statements:
- width = st.end_line - st.start_line
- for line in range(st.start_line, st.end_line + 1):
- held = best.get(line)
- if held is None or width < held[0]:
- best[line] = (width, block.id, st.start_line)
- return {line: (bid, start) for line, (_w, bid, start) in best.items()}
-
-
-def _infer_in_out(
- def_lines: dict[str, list[int]],
- use_lines: dict[str, list[int]],
- s: int,
- e: int,
- declared_first: dict[str, frozenset[int]],
- *,
- reads: dict[str, list[tuple[int, frozenset[int]]]],
- valueless: dict[str, frozenset[int]] | None = None,
-) -> tuple[tuple[str, ...], tuple[str, ...]]:
- """Infer IN (parameters) and OUT (return) variables for span ``[s, e]``.
-
- IN: a variable read in the span whose first in-span read is not preceded by
- an in-span write, and which has a definition before the span (a parameter or
- an earlier assignment). A write on the same line as that read precedes it
- only where *declared_first* (:func:`_declared_before_read`) says the line
- declares the name first. An earlier in-span write covers a later read only
- on the paths that run it, so a variable is also IN when some in-span read
- of it may still observe a definition from before the span (*reads*,
- :func:`_reads_observing`): the write sits under an ``if``, in a loop
- body, or in an arm. A declaration that gives the name no value
- (*valueless*, :func:`_valueless_lines`) is not such a definition: the
- compiler has proved the read assigned where the may-analysis cannot, and
- a call that passed the name would not compile. OUT: a variable written in
- the span with a read after it that may observe one of those writes
- (*reads* again). A read before the span that a write in it reaches around
- a loop is not an OUT: :func:`_loop_carry_free` refuses those spans. Both
- line lists are sorted (:func:`_var_lines`), so the span tests are bisects.
- """
- params: list[str] = []
- returns: list[str] = []
- for var in sorted(set(def_lines) | set(use_lines)):
- dl = def_lines.get(var, [])
- ul = use_lines.get(var, [])
- u_lo, u_hi = bisect_left(ul, s), bisect_right(ul, e)
- d_lo, d_hi = bisect_left(dl, s), bisect_right(dl, e)
-
- if u_lo < u_hi and d_lo > 0:
- first_use = ul[u_lo]
- declaring = declared_first.get(var, ())
- bare = valueless.get(var, ()) if valueless else ()
- uncovered = first_use not in declaring and not (d_lo < d_hi and dl[d_lo] < first_use)
- if uncovered or any(
- s <= ln <= e
- and ln not in declaring
- and any(d < s and d not in bare for d in seen)
- for ln, seen in reads.get(var, ())
- ):
- params.append(var)
-
- if d_lo < d_hi and any(
- ln > e and any(s <= d <= e for d in seen) for ln, seen in reads.get(var, ())
- ):
- returns.append(var)
- return tuple(params), tuple(returns)
-
-
-class _SharedState(NamedTuple):
- """Per variable a closure shares with the function, sorted line lists:
- the closures' reads and writes (``Captured.shared`` / ``.writes``), the
- function's own defs, all its references, and its binding declarations."""
-
- reads: dict[str, list[int]]
- writes: dict[str, list[int]]
- defs: dict[str, list[int]]
- refs: dict[str, list[int]]
- binds: dict[str, frozenset[int]]
-
-
-def _closure_state(
- def_use: FunctionDefUse,
- def_lines: dict[str, list[int]],
- use_lines: dict[str, list[int]],
-) -> _SharedState | None:
- """The closure-shared locals of a function, or None when it has none
- (then the gate costs nothing per span)."""
- reads: dict[str, list[int]] = defaultdict(list)
- writes: dict[str, list[int]] = defaultdict(list)
- for out, occurrences in (
- (reads, def_use.captured.shared),
- (writes, def_use.captured.writes),
- ):
- for occ in occurrences:
- if occ.name in def_lines: # a name this function binds
- out[occ.name].append(occ.line)
- if not reads and not writes:
- return None
- names = set(reads) | set(writes)
- return _SharedState(
- reads={var: sorted(lines) for var, lines in reads.items()},
- writes={var: sorted(lines) for var, lines in writes.items()},
- defs={var: def_lines.get(var, []) for var in names},
- refs={var: sorted((*def_lines.get(var, ()), *use_lines.get(var, ()))) for var in names},
- binds=_declaration_lines(def_use, binding=True),
- )
-
-
-def _count_in(lines: list[int], s: int, e: int) -> int:
- return bisect_right(lines, e) - bisect_left(lines, s)
-
-
-def _closure_state_crosses(
- st: _SharedState, span: list[Node], s: int, e: int, block: Node, lmap: LanguageNodeMap
-) -> bool:
- """True when a local a closure shares crosses the span boundary.
-
- A closure runs when it is called, not where it is written, so line
- liveness cannot place its reads and writes; an IN/OUT signature copies a
- value where the closure needs the variable itself. Refused:
-
- - a closure in the span writes a local anything outside the span refers
- to (code, or a closure there): the lifted closure writes the helper's
- copy. A counter declared in the span with its ``inc`` closure is
- refused this way when the count is read after the span;
- - a closure outside the span writes a local the span refers to: the span
- reads or writes a stale copy;
- - the span writes a local that a closure written above it reads: the
- closure runs later and sees the old value. Measured on hermes
- ``apps/desktop/src/lib/ansi.ts::parseAnsi``, whose ``pushText`` reads
- the ``bold`` / ``fg`` a span inside the loop sets.
-
- The last two are waived when the span's name is a binding of its own
- (:func:`_span_binds`, :func:`_span_only_rebinds`): the closure's variable
- is then another one.
- """
- for var in st.reads.keys() | st.writes.keys():
- writes = st.writes.get(var, [])
- reads = st.reads.get(var, [])
- refs = st.refs[var]
- w_in = _count_in(writes, s, e)
- if w_in and (
- w_in < len(writes)
- or _count_in(reads, s, e) < len(reads)
- or _count_in(refs, s, e) < len(refs)
- ):
- return True
- stale = w_in < len(writes) and _count_in(refs, s, e) > 0
- if stale and not _span_binds(st, var, span, block, lmap):
- return True
- overwritten = bool(reads) and reads[0] < s and _count_in(st.defs[var], s, e) > 0
- if overwritten and not _span_only_rebinds(st, var, s, e, block, lmap):
- return True
- return False
-
-
-def _declares_in(node: Node, lines: frozenset[int], lmap: LanguageNodeMap) -> bool:
- """True when *node* is a declaration statement holding one of *lines*."""
- lo, hi = node.start_point[0] + 1, node.end_point[0] + 1
- return node.type in lmap.local_decl_kinds and any(lo <= ln <= hi for ln in lines)
-
-
-def _declared_above(binds: frozenset[int], block: Node, s: int, lmap: LanguageNodeMap) -> bool:
- """True when a statement of *block* above the span already declares the
- name: a re-declaration in the same scope (Go's ``v, err := h()``) is an
- assignment to it, not a new binding."""
- return any(
- node.end_point[0] + 1 < s and _declares_in(node, binds, lmap)
- for node in block.named_children
- )
-
-
-def _span_binds(
- st: _SharedState, var: str, span: list[Node], block: Node, lmap: LanguageNodeMap
-) -> bool:
- """True when one of the span's own top-level statements declares a new
- binding of *var*, so every reference to it in the span is to that one.
-
- Top-level only: a declaration in a nested block shadows just that block.
- """
- binds = st.binds.get(var, frozenset())
- s = span[0].start_point[0] + 1
- return (
- bool(binds)
- and not _declared_above(binds, block, s, lmap)
- and any(_declares_in(node, binds, lmap) for node in span)
- )
-
-
-def _span_only_rebinds(
- st: _SharedState, var: str, s: int, e: int, block: Node, lmap: LanguageNodeMap
-) -> bool:
- """True when every write the span makes to *var* creates a new binding
- (a declaration or a loop binder, at any depth), so none of them reaches
- the variable a closure outside the span reads."""
- binds = st.binds.get(var, frozenset())
- defs = st.defs[var][bisect_left(st.defs[var], s) : bisect_right(st.defs[var], e)]
- return all(ln in binds for ln in defs) and not _declared_above(binds, block, s, lmap)
-
-
-def _holds_a_named_nested_function(span: list[Node], lmap: LanguageNodeMap) -> bool:
- """True when the span contains a nested function that binds its own name.
-
- Def/use is computed per function and deliberately does not descend into a
- nested scope, so a sibling closure's call to such a helper is invisible
- here. Lifting the declaration out of the scope moves the binding away from
- those callers, and nothing in the facts would show it.
-
- A named nested function exists to be called from elsewhere in its scope, so
- the safe answer is to refuse rather than to guess at its callers. An
- anonymous lambda bound to a local is a value and stays governed by the
- ordinary IN/OUT rules. Measured on this repo's ranked head:
- ``vscode/src/features/changeIntel.ts::registerChangeIntel`` offered its whole
- ``render`` declaration while a sibling closure called it.
- """
- kinds = lmap.function_kinds
- if not kinds:
- return False
- for st in span:
- stack = [st]
- while stack:
- node = stack.pop()
- if node.type in kinds and node.child_by_field_name("name") is not None:
- return True
- stack.extend(node.children)
- return False
-
-
-def _hoisted_bindings(
- def_lines: dict[str, list[int]],
- use_lines: dict[str, list[int]],
-) -> list[tuple[int, int]]:
- """First-definition lines of names the function reads before defining them.
-
- A read before the only definition of a name can only work by hoisting: a
- JS/TS ``function foo()`` declaration is visible from the top of its scope,
- so code above it calls it. A span holding that definition cannot be lifted -
- an OUT cannot express it, because the value has to exist *before* the span
- runs, not after - so :func:`find_extractions` refuses any span containing
- one of these lines.
-
- Returns ``(first_def, first_use)`` pairs. A span refuses when its range holds
- a ``first_def`` whose ``first_use`` sits *above* the span: the earlier read
- has no earlier definition to answer it, so the span holds the only one.
- A read that sits inside the span, before the definition, is a different
- shape and is left to the loop-carried gate.
-
- Computed once per function because the shape is rare (usually no name
- qualifies at all), and asking per candidate span meant walking every
- variable thousands of times. Both lists are sorted by :func:`_var_lines`,
- so the first entry of each is the earliest.
-
- Measured on this repo's ranked head:
- ``vscode/src/features/changeIntel.ts::registerChangeIntel`` offered its whole
- ``render`` declaration as a span while ``render(partners)`` sat above it.
- """
- hoisted: list[tuple[int, int]] = []
- for var, defs in def_lines.items():
- uses = use_lines.get(var)
- if defs and uses and uses[0] < defs[0]:
- hoisted.append((defs[0], uses[0]))
- return sorted(hoisted)
-
-
-def _declaration_lines(
- def_use: FunctionDefUse, *, binding: bool = False
-) -> dict[str, frozenset[int]]:
- """Per variable, the lines that declare it (a ``let`` / ``var`` / ``:=`` /
- typed local, as the dialect marks with ``declared_at``, or an import).
- With *binding*, the lines that create a new binding instead
- (``declares``: multi-line declarators and loop binders included, a TS/JS
- ``var`` not)."""
- lines: dict[str, set[int]] = defaultdict(set)
- for d in def_use.definitions:
- if (d.declares if binding else d.declared_at is not None or d.imports):
- lines[d.var].add(d.line)
- return {var: frozenset(found) for var, found in lines.items()}
-
-
-def _declaration_escapes(
- s: int,
- e: int,
- returns: tuple[str, ...],
- decl_lines: dict[str, frozenset[int]],
- def_lines: dict[str, list[int]],
- use_lines: dict[str, list[int]],
- declared_first: dict[str, frozenset[int]],
-) -> bool:
- """True when the span declares a name the code after it still refers to.
-
- A returned name is fine: the caller declares it from the helper's result.
- Otherwise the declaration leaves with the span and the caller's next
- reference names nothing, even when that reference is a plain assignment
- (Go ``var out T`` in the span, ``out = x`` after it, which is why liveness
- did not make it an OUT). A later reference that declares the name afresh
- (``y, err := g()``, a second ``for i := ...``, whose reads on that line
- follow the declaration per *declared_first*) needs nothing from the span.
- Block scopes are not modelled, so a same-named variable of an outer scope
- read after the span also refuses: a missed span, never a broken one.
- """
- return any(
- var not in returns
- and any(s <= ln <= e for ln in declared)
- and _needed_after(var, e, declared, def_lines, use_lines, declared_first)
- for var, declared in decl_lines.items()
- )
-
-
-def _bindings_escape(
- s: int,
- e: int,
- returns: tuple[str, ...],
- decl_lines: dict[str, frozenset[int]],
- def_lines: dict[str, list[int]],
- use_lines: dict[str, list[int]],
- declared_first: dict[str, frozenset[int]],
- outer: dict[str, frozenset[int]],
-) -> bool:
- """A binding the span cannot take along: a declaration the code after it
- still needs (:func:`_declaration_escapes`), or a ``global`` / ``nonlocal``
- write whose declaration stays behind (:func:`_outer_write_escapes`)."""
- return _declaration_escapes(
- s, e, returns, decl_lines, def_lines, use_lines, declared_first
- ) or _outer_write_escapes(s, e, outer, def_lines)
-
-
-def _outer_write_escapes(
- s: int, e: int, outer: dict[str, frozenset[int]], def_lines: dict[str, list[int]]
-) -> bool:
- """True when the span writes a ``global`` / ``nonlocal`` name it cannot
- take its declaration along for: one declared outside the span, or also
- written outside it. In a helper without the declaration the write binds a
- local and the module value never changes."""
- for var, declared in outer.items():
- writes = def_lines.get(var, ())
- if not any(s <= ln <= e for ln in writes):
- continue
- if any(not s <= ln <= e for ln in (*declared, *writes)):
- return True
- return False
-
-
-def _needed_after(
- var: str,
- e: int,
- declared: frozenset[int],
- def_lines: dict[str, list[int]],
- use_lines: dict[str, list[int]],
- declared_first: dict[str, frozenset[int]],
-) -> bool:
- """True when the first reference to *var* after line *e* relies on an
- earlier declaration: it is a plain use or assignment, not a fresh one."""
- uses = use_lines.get(var, ())
- after = [ln for ln in (*def_lines.get(var, ()), *uses) if ln > e]
- if not after:
- return False
- first = min(after)
- if first not in declared:
- return True
- return first in uses and first not in declared_first.get(var, ())
-
-
-def _outs_definitely_assigned(
- span: list[Node],
- returns: tuple[str, ...],
- def_lines: dict[str, list[int]],
- lmap: LanguageNodeMap,
-) -> bool:
- """True when every OUT variable is written on *every* path through the span.
-
- A conditionally written OUT is the unsound case line-based liveness cannot
- see: the helper returns the unwritten value on the paths that skip the
- write, and the caller's assignment then clobbers the live one. The proof
- needs branch structure, so it runs structurally here and the span is
- refused whenever the proof does not go through.
- """
- return all(_stmts_assign(span, var, def_lines.get(var, []), lmap) for var in returns)
-
-
-def _stmts_assign(stmts: list[Node], var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:
- """True when *stmts* writes *var* on every path through them."""
- return any(_stmt_assigns(st, var, defs, lmap) for st in stmts)
-
-
-def _stmt_assigns(st: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:
- """True only when *st* is proved to write *var* however it is entered.
-
- Positive proof only: a statement kind this cannot classify returns False.
- Defaulting to True would let any statement sharing a line with a
- conditional write stand in as the proof.
- """
- lo = st.start_point[0] + 1
- hi = st.end_point[0] + 1
- if not any(lo <= d <= hi for d in defs):
- return False
- if _is_conditional(st, lmap):
- return _conditional_assigns(st, var, defs, lmap)
- if st.type in lmap.block_kinds:
- container = _unwrap_container(st, lmap.block_kinds)
- return _stmts_assign(container.named_children, var, defs, lmap)
- if st.type in lmap.with_kinds or st.type in lmap.statement_wrapper_kinds:
- return _stmts_assign(list(st.named_children), var, defs, lmap)
- return _unconditional_write(st, var, lmap)
-
-
-def _unconditional_write(st: Node, var: str, lmap: LanguageNodeMap) -> bool:
- """True when *st* contains a write to *var* no branch or loop guards.
-
- The walk stops at any control container (a loop body may run zero times, a
- branch arm may not be taken, a catch arm may not fire) and at nested
- scopes, so only writes on the statement's own straight-line path count.
- """
- write_kinds = lmap.assignment_kinds | lmap.augmented_assign_kinds | lmap.local_decl_kinds
- if not write_kinds:
- return False
- barriers = (
- lmap.loop_kinds
- | lmap.try_kinds
- | lmap.catch_kinds
- | lmap.switch_kinds
- | lmap.case_kinds
- | lmap.branch_kinds
- | lmap.if_kinds
- | lmap.function_kinds
- | lmap.lambda_kinds
- )
- if st.type in barriers:
- return False
- stack = [st]
- while stack:
- node = stack.pop()
- if node.id != st.id and node.type in barriers:
- continue
- if node.type in write_kinds and _writes_var(node, var):
- return True
- stack.extend(node.children)
- return False
-
-
-def _writes_var(node: Node, var: str) -> bool:
- """True when *var* is on the binding side of this write node."""
- for field in ("left", "pattern", "declarator", "name"):
- target = node.child_by_field_name(field)
- if target is not None:
- return var in _identifiers(target)
- return var in _identifiers(node)
-
-
-def _is_conditional(node: Node, lmap: LanguageNodeMap) -> bool:
- return node.type in lmap.if_kinds or (
- node.type in lmap.branch_kinds and node.child_by_field_name("condition") is not None
- )
-
-
-def _conditional_assigns(node: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:
- """True when a conditional chain is exhaustive and every arm writes *var*.
-
- Two chain shapes. Python flattens ``elif``/``else`` into repeated
- ``alternative`` fields on one ``if``, so the terminal else is a sibling of
- the chained arms. The C family nests instead: the single ``alternative`` is
- another ``if``, whose own else ends the chain. Both must be exhaustive.
- """
- consequence = node.child_by_field_name("consequence") or node.child_by_field_name("body")
- if consequence is None:
- return False
- alternatives = list(node.children_by_field_name("alternative"))
- if not alternatives:
- return False # no else at all: some path skips the write
- if not _branch_assigns(consequence, var, defs, lmap):
- return False
- for alt in alternatives:
- if _is_conditional(alt, lmap):
- # Nested chain (C family): it carries the chain's terminal else.
- if len(alternatives) == 1:
- return _conditional_assigns(alt, var, defs, lmap)
- # Flattened chain (Python's ``elif``): the else is a sibling below.
- arm = alt.child_by_field_name("consequence") or alt.child_by_field_name("body")
- if arm is None or not _branch_assigns(arm, var, defs, lmap):
- return False
- elif not _branch_assigns(alt, var, defs, lmap):
- return False
- return any(not _is_conditional(alt, lmap) for alt in alternatives)
-
-
-def _branch_assigns(node: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:
- if _is_conditional(node, lmap):
- return _conditional_assigns(node, var, defs, lmap)
- if node.type in lmap.block_kinds:
- container = _unwrap_container(node, lmap.block_kinds)
- return _stmts_assign(container.named_children, var, defs, lmap)
- return _stmts_assign(list(node.named_children), var, defs, lmap)
-
-
-def _rebinding_defs_dropped(
- analysis: FunctionAnalysis, def_lines: dict[str, list[int]], lmap: LanguageNodeMap
-) -> dict[str, list[int]]:
- """*def_lines* without the names exception handlers bind on their head
- line (``except E as e:``, ``catch (err)``). Such a binding never reaches
- the next iteration: Python unbinds it at the handler's end and a catch
- parameter is scoped to its block, so a read of the name elsewhere in the
- loop is another handler's own."""
- heads = [
- st.node
- for block in analysis.cfg.blocks
- for st in block.statements
- if st.is_head and st.kind in lmap.catch_kinds and st.node is not None
- ]
- if not heads:
- return def_lines
- bound: set[tuple[str, int]] = set()
- for head in heads:
- # Only the clause's own binder: before its body, so a one-line body's
- # write on the same line still counts.
- body = next((c for c in head.named_children if c.type in lmap.block_kinds), None)
- end = body.start_point if body is not None else head.end_point
- bound |= {
- (d.var, d.line)
- for d in analysis.def_use.definitions
- if head.start_point <= (d.line - 1, d.column) < end
- }
- return {var: [ln for ln in lines if (var, ln) not in bound] for var, lines in def_lines.items()}
-
-
-def _loop_carry_free(
- span: list[Node],
- loop: Node,
- s: int,
- e: int,
- def_lines: dict[str, list[int]],
- use_lines: dict[str, list[int]],
- lmap: LanguageNodeMap,
-) -> bool:
- """True when a span nested in *loop* carries no state between iterations.
-
- Two shapes are refused. A variable the span writes that the loop also reads
- without the span having written it first is loop-carried: that read takes
- the previous iteration's value, and lifting the write into a helper whose
- result is discarded silently drops it.
-
- The read does not have to sit above the span. ``clojure.py::_spec_namespaces``
- reads ``expect_ns`` and then writes it, both inside one candidate span, in a
- ``while`` loop: textually the read precedes the write, so nothing follows
- the span to make the variable an OUT, and the state machine breaks on the
- next iteration. Testing only for reads above the span missed it, so the test
- is "read with no in-span write before it", which subsumes the old one.
-
- Also refused: a call on a name the loop header reads, which mutates the
- state the loop iterates over (``entries.remove(entry)`` inside
- ``for entry in entries``), and which an IN/OUT signature cannot express.
- """
- loop_start = loop.start_point[0] + 1
- for var, lines in def_lines.items():
- in_span_writes = [ln for ln in lines if s <= ln <= e]
- if not in_span_writes:
- continue
- for read in use_lines.get(var, []):
- if not loop_start <= read <= e:
- continue
- if not any(write < read for write in in_span_writes):
- return False
-
- carried = _loop_header_names(loop, lmap)
- if not carried:
- return True
- scope_kinds = lmap.function_kinds | lmap.lambda_kinds
- for st in span:
- for call in _descend(st, lmap.call_kinds, scope_kinds):
- callee = call.child_by_field_name("function")
- if callee is None:
- named = call.named_children
- callee = named[0] if named else None
- root = _receiver_root(callee) if callee is not None else None
- if root is not None and root in carried:
- return False
- return True
-
-
-def _receiver_root(node: Node) -> str | None:
- """The base name a call is made on: ``entries`` in ``entries.remove(x)``.
-
- Only the root counts. Matching any name in the callee read
- ``existing.pages.push(page)`` as touching the ``pages`` the loop iterates,
- when the receiver is ``existing`` and the collision is in a member name.
- """
- cur = node
- while True:
- nxt = None
- for field in ("object", "operand", "value", "argument", "function"):
- child = cur.child_by_field_name(field)
- if child is not None:
- nxt = child
- break
- if nxt is None:
- break
- cur = nxt
- if not cur.children and cur.type.endswith("identifier"):
- text = cur.text
- return text.decode("utf-8", "replace") if text else None
- return None
-
-
-# Fields a loop header binds through. Read structurally: a line-range test
-# cannot tell the header's binder from a variable reassigned on the body's
-# first line, and getting that wrong drops the iterated collection out of the
-# carried set, which is exactly the name the mutation check exists to catch.
-_BINDER_FIELDS = ("left", "pattern", "declarator", "name")
-
-
-def _loop_header_names(loop: Node, lmap: LanguageNodeMap) -> set[str]:
- """Names the loop header reads, minus the ones it binds (the loop variable)."""
- body = loop.child_by_field_name("body")
- names: set[str] = set()
- # The binder is a field of the loop itself in most grammars, and of a
- # header clause it wraps in Go's ``range``.
- bound: set[str] = _binder_names(loop)
- for child in loop.children:
- if body is not None and child.id == body.id:
- continue
- names |= _identifiers(child)
- bound |= _binder_names(child)
- return names - bound
-
-
-def _binder_names(node: Node) -> set[str]:
- """Identifiers in the binding position of *node* or of a header clause it
- wraps (Go nests its range clause inside the ``for``)."""
- out: set[str] = set()
- for field in _BINDER_FIELDS:
- target = node.child_by_field_name(field)
- if target is not None:
- out |= _identifiers(target)
- if not out:
- for child in node.named_children:
- for field in _BINDER_FIELDS:
- target = child.child_by_field_name(field)
- if target is not None:
- out |= _identifiers(target)
- return out
-
-
-def _identifiers(node: Node) -> set[str]:
- out: set[str] = set()
- stack = [node]
- while stack:
- cur = stack.pop()
- if not cur.children and cur.type.endswith("identifier"):
- text = cur.text
- if text:
- out.add(text.decode("utf-8", "replace"))
- stack.extend(cur.children)
- return out
-
-
-def _descend(root: Node, kinds: frozenset[str], skip: frozenset[str]) -> list[Node]:
- found: list[Node] = []
- stack = [root]
- while stack:
- cur = stack.pop()
- if cur.type in kinds:
- found.append(cur)
- for child in cur.children:
- if child.type in skip:
- continue
- stack.append(child)
- return found
-
-
-def _unwrap_container(node: Node, block_kinds: frozenset[str]) -> Node:
- """Descend through a single nested statement-container (Go's
- ``block`` -> ``statement_list``) to the node whose named children are the
- actual statements; returns *node* unchanged when it is already that node."""
- cur = node
- while True:
- named = cur.named_children
- if len(named) == 1 and named[0].type in block_kinds:
- cur = named[0]
- else:
- return cur
-
-
-def all_blocks(
- fn_node: Node,
- block_kinds: frozenset[str],
- scope_kinds: frozenset[str],
- loop_kinds: frozenset[str],
-) -> list[tuple[Node, Node | None]]:
- """Every statement container in the function (body + nested), excluding the
- bodies of nested functions / lambdas, each paired with the innermost loop
- enclosing it (``None`` at loop-free depth)."""
- body = fn_node.child_by_field_name("body")
- if body is None:
- return []
- blocks: list[tuple[Node, Node | None]] = []
- stack: list[tuple[Node, Node | None]] = [(body, None)]
- while stack:
- node, loop = stack.pop()
- if node.type in block_kinds:
- blocks.append((node, loop))
- is_loop = node.type in loop_kinds
- body = node.child_by_field_name("body") if is_loop else None
- for child in node.children:
- if child.type in scope_kinds:
- continue
- # Only the loop's body repeats. A ``for ... else`` clause runs once.
- in_loop = node if (is_loop and body is not None and child.id == body.id) else loop
- stack.append((child, in_loop))
- return blocks
-
-
-def _exit_macros(lmap: LanguageNodeMap) -> tuple[frozenset[str], frozenset[str]]:
- """The macro node kinds and the macro names that exit the function."""
- return lmap.exit_macro_kinds, lmap.exit_macro_names
-
-
-def _awaits(lmap: LanguageNodeMap) -> tuple[frozenset[str], frozenset[str]]:
- """The await tokens, and the nodes that own the awaits inside them."""
- return lmap.await_kinds, lmap.await_scope_kinds
-
-
def _span_kinds(scan: _Scan) -> FactKinds:
await_kinds, await_scope_kinds = scan.awaits
return FactKinds(
@@ -1495,95 +636,3 @@ def _push_children(
t = child.type
if t not in scan.scopes or (scan.receiver is not None and t in scan.lambdas):
stack.append((child, scope))
-
-
-# Index access on a receiver field (``self.cache[k] = v``) assigns into it.
-_INDEX_KINDS = frozenset({"subscript", "subscript_expression", "index_expression", "array_access"})
-# Unpacking targets whose named children are targets themselves
-# (``a, self.b = ...``, ``[this.x, y] = ...``).
-_UNPACK_KINDS = frozenset(
- {
- "pattern_list",
- "tuple_pattern",
- "list_pattern",
- "expression_list",
- "tuple",
- "list",
- "array_pattern",
- "object_pattern",
- "parenthesized_expression",
- "list_splat_pattern",
- "rest_pattern",
- }
-)
-# Marks a target rooted at the receiver whose shape is not read, so the
-# span's assigned fields are unknown.
-_UNREAD_TARGET = ""
-
-
-def _receiver_ref(node: Node, receiver: Receiver, assigns: set[str]) -> bool:
- """True when *node* names the receiver; an assignment to one of its
- fields adds the field's name to *assigns* (the receiver leaf under it is
- visited next)."""
- if node.type in receiver.write_kinds:
- target = (
- node.child_by_field_name("left")
- or node.child_by_field_name("argument")
- or (node.named_children[0] if node.named_children else None)
- )
- if target is not None:
- _collect_assigned(target, receiver, assigns)
- return False
- return _is_receiver(node, receiver)
-
-
-def _collect_assigned(target: Node, receiver: Receiver, assigns: set[str]) -> None:
- """The receiver fields one assignment target writes, unpacking included;
- a target of another shape that mentions the receiver makes them unknown."""
- if target.type in _UNPACK_KINDS:
- for each in target.named_children:
- _collect_assigned(each, receiver, assigns)
- return
- if target.type in ("assignment_pattern", "object_assignment_pattern"): # ``[this.a = 1]``
- left = target.child_by_field_name("left")
- if left is not None:
- _collect_assigned(left, receiver, assigns)
- return
- if target.type == "pair_pattern": # ``{key: this.a}``
- value = target.child_by_field_name("value")
- if value is not None:
- _collect_assigned(value, receiver, assigns)
- return
- field = _receiver_field(target, receiver)
- if field:
- assigns.add(field)
- elif target.type not in receiver.access_kinds | _INDEX_KINDS and mentions_receiver(
- target, receiver.names
- ):
- assigns.add(_UNREAD_TARGET)
-
-
-def _is_receiver(node: Node, receiver: Receiver) -> bool:
- """A leaf naming the receiver: a ``this`` / ``self`` / ``super`` token,
- or an identifier spelling a named receiver (Python ``self``, Go ``s``)
- or ``super`` (Python's no-argument ``super()``)."""
- if node.children or not node.text:
- return False
- if node.type in receiver.names or node.type == SUPER:
- return True
- if node.type != "identifier":
- return False
- text = node.text.decode("utf-8", "replace")
- return text in receiver.names or text == SUPER
-
-
-def _receiver_field(target: Node, receiver: Receiver) -> str | None:
- """``x`` for an assignment target ``self.x``, ``self.x.y`` or ``self.x[k]``."""
- cur: Node | None = target
- while cur is not None and (cur.type in receiver.access_kinds or cur.type in _INDEX_KINDS):
- if cur.type in receiver.access_kinds:
- field = self_member_name(cur, receiver.names)
- if field is not None:
- return field
- cur = member_object(cur)
- return None
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice_liveness.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice_liveness.py
new file mode 100644
index 000000000..bb67e403a
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice_liveness.py
@@ -0,0 +1,350 @@
+"""Line-based liveness over def/use facts: a span's IN / OUT and the bindings it cannot take along."""
+
+from __future__ import annotations
+
+from bisect import bisect_left, bisect_right
+from collections import defaultdict
+from typing import TYPE_CHECKING
+
+from .reaching import definitions_at, observed_definitions
+
+if TYPE_CHECKING:
+ from collections.abc import Container
+
+ from .analyze import FunctionAnalysis
+ from .cfg import CFG
+ from .defuse import FunctionDefUse
+
+
+def _var_lines(def_use: FunctionDefUse) -> tuple[dict[str, list[int]], dict[str, list[int]]]:
+ """Per-variable sorted def lines and use lines from the def/use facts.
+
+ Parameter definitions are included (seeded at the signature line), so a
+ parameter naturally counts as "defined before" any body span. Reads inside
+ nested closures (``def_use.captured.reads``) count as uses at their own line.
+ An import is not a def here: a helper re-imports a name rather than take
+ or return it, and :func:`_declaration_escapes` refuses a span holding an
+ import that later code still reads.
+ """
+ def_lines: dict[str, list[int]] = defaultdict(list)
+ use_lines: dict[str, list[int]] = defaultdict(list)
+ imported: set[str] = set()
+ for d in def_use.definitions:
+ if d.imports:
+ imported.add(d.var)
+ else:
+ def_lines[d.var].append(d.line)
+ for bdu in def_use.blocks.values():
+ for u in bdu.uses:
+ # A may-def's paired use is bookkeeping, not a read: counted, a
+ # binder declared inside a ``match`` arm became its own parameter.
+ if not u.echo:
+ use_lines[u.name].append(u.line)
+ # A closure's read counts where the closure is written: lifting the code
+ # around it moves the read with it. Only names this function binds matter.
+ for u in def_use.captured.reads:
+ if u.name in def_lines or u.name in imported:
+ use_lines[u.name].append(u.line)
+ for lines in def_lines.values():
+ lines.sort()
+ for lines in use_lines.values():
+ lines.sort()
+ return def_lines, use_lines
+
+
+def _import_reads(analysis: FunctionAnalysis) -> tuple[tuple[int, int], ...]:
+ """Function-local import lines paired with the reads they reach."""
+ imports = {
+ (definition.var, definition.line)
+ for definition in analysis.reaching.definitions
+ if definition.imports
+ }
+ if not imports:
+ return ()
+ import_lines: dict[str, list[int]] = defaultdict(list)
+ for name, line in sorted(imports):
+ import_lines[name].append(line)
+ return tuple(
+ (definition_line, use_line)
+ for name, reads in _reads_observing(analysis, import_lines).items()
+ for use_line, seen in reads
+ for definition_line in seen
+ if (name, definition_line) in imports
+ )
+
+
+def _declared_before_read(def_use: FunctionDefUse) -> dict[str, frozenset[int]]:
+ """Per variable, the lines where a declaration of it comes before its first
+ read on the same line.
+
+ Lines alone cannot order a write and a read that share one. A C-style
+ ``for (int i = 0; i < n; i++)`` declares ``i`` and then reads it, while
+ ``total = total + a[i]`` reads ``total`` and then writes it. Only a
+ declaration is ordered here, by where its declarator ends: a read past that
+ point sees the new name, and a read inside the declaration's own
+ initializer (Go's ``x := x + 1`` in an inner scope) still sees the outer
+ one. A plain assignment keeps the line rule.
+ """
+ first_read: dict[tuple[str, int], int] = {}
+ for bdu in def_use.blocks.values():
+ for u in bdu.uses:
+ if u.echo:
+ continue
+ key = (u.name, u.line)
+ first_read[key] = min(u.column, first_read.get(key, u.column))
+ declared: dict[str, set[int]] = defaultdict(set)
+ for d in def_use.definitions:
+ # No read on the line sorts before any declarator, so it never matches.
+ if d.declared_at is not None and d.declared_at <= first_read.get((d.var, d.line), -1):
+ declared[d.var].add(d.line)
+ return {var: frozenset(lines) for var, lines in declared.items()}
+
+
+def _valueless_lines(def_use: FunctionDefUse) -> dict[str, frozenset[int]]:
+ """Per variable, the lines whose only definitions of it are declarations
+ that give it no value (``Definition.valueless``): Java's ``List<T> xs;``,
+ TypeScript's ``let x: T;``. A line that also assigns the name
+ (``int x; x = 1;``) is not one: what a read sees from it is the value."""
+ bare: dict[str, set[int]] = defaultdict(set)
+ valued: dict[str, set[int]] = defaultdict(set)
+ for d in def_use.definitions:
+ (bare if d.valueless else valued)[d.var].add(d.line)
+ return {var: frozenset(at - valued[var]) for var, at in bare.items()}
+
+
+def _reads_observing(
+ analysis: FunctionAnalysis, def_lines: dict[str, list[int]]
+) -> dict[str, list[tuple[int, frozenset[int]]]]:
+ """Per variable, each read's line with the def lines it may observe.
+
+ Reads take reaching definitions over the CFG, so a later write on a path
+ that need not run (an ``else`` arm, a ``try`` body that may raise first)
+ does not hide the span's value from the read. A closure's read observes
+ what reaches the statement that creates the closure, or, where no recorded
+ statement covers it, the nearest def above it: a read is never dropped,
+ since a dropped read can only drop a return. Only names in *def_lines*
+ count for closures, as in :func:`_var_lines`.
+ """
+ def_use, reaching = analysis.def_use, analysis.reaching
+ defs = reaching.definitions
+ reads: dict[str, list[tuple[int, frozenset[int]]]] = defaultdict(list)
+ for use, seen in observed_definitions(def_use, reaching):
+ if not use.echo:
+ reads[use.name].append((use.line, frozenset(defs[i].line for i in seen)))
+ statements: dict[int, tuple[int, int]] | None = None
+ for u in def_use.captured.reads:
+ dl = def_lines.get(u.name)
+ if not dl:
+ continue
+ if statements is None:
+ statements = _statements_by_line(analysis.cfg)
+ at = statements.get(u.line)
+ if at is not None:
+ observed = frozenset(
+ defs[i].line for i in definitions_at(def_use, reaching, at[0], u.name, at[1])
+ )
+ else:
+ k = bisect_left(dl, u.line)
+ observed = frozenset(dl[k - 1 : k])
+ reads[u.name].append((u.line, observed))
+ return reads
+
+
+def _statements_by_line(cfg: CFG) -> dict[int, tuple[int, int]]:
+ """Per line, (block id, start line) of the innermost recorded statement
+ covering it."""
+ best: dict[int, tuple[int, int, int]] = {}
+ for block in cfg.blocks:
+ for st in block.statements:
+ width = st.end_line - st.start_line
+ for line in range(st.start_line, st.end_line + 1):
+ held = best.get(line)
+ if held is None or width < held[0]:
+ best[line] = (width, block.id, st.start_line)
+ return {line: (bid, start) for line, (_w, bid, start) in best.items()}
+
+
+def _infer_in_out(
+ def_lines: dict[str, list[int]],
+ use_lines: dict[str, list[int]],
+ s: int,
+ e: int,
+ declared_first: dict[str, frozenset[int]],
+ *,
+ reads: dict[str, list[tuple[int, frozenset[int]]]],
+ valueless: dict[str, frozenset[int]] | None = None,
+) -> tuple[tuple[str, ...], tuple[str, ...]]:
+ """Infer IN (parameters) and OUT (return) variables for span ``[s, e]``.
+
+ IN: a variable read in the span whose first in-span read is not preceded by
+ an in-span write, and which has a definition before the span (a parameter or
+ an earlier assignment). A write on the same line as that read precedes it
+ only where *declared_first* (:func:`_declared_before_read`) says the line
+ declares the name first. An earlier in-span write covers a later read only
+ on the paths that run it, so a variable is also IN when some in-span read
+ of it may still observe a definition from before the span (*reads*,
+ :func:`_reads_observing`): the write sits under an ``if``, in a loop
+ body, or in an arm. A declaration that gives the name no value
+ (*valueless*, :func:`_valueless_lines`) is not such a definition: the
+ compiler has proved the read assigned where the may-analysis cannot, and
+ a call that passed the name would not compile. OUT: a variable written in
+ the span with a read after it that may observe one of those writes
+ (*reads* again). A read before the span that a write in it reaches around
+ a loop is not an OUT: :func:`_loop_carry_free` refuses those spans. Both
+ line lists are sorted (:func:`_var_lines`), so the span tests are bisects.
+ """
+ params: list[str] = []
+ returns: list[str] = []
+ for var in sorted(set(def_lines) | set(use_lines)):
+ dl = def_lines.get(var, [])
+ ul = use_lines.get(var, [])
+ u_lo, u_hi = bisect_left(ul, s), bisect_right(ul, e)
+ d_lo, d_hi = bisect_left(dl, s), bisect_right(dl, e)
+
+ if u_lo < u_hi and d_lo > 0:
+ first_use = ul[u_lo]
+ declaring = declared_first.get(var, ())
+ covered = first_use in declaring or (d_lo < d_hi and dl[d_lo] < first_use)
+ bare = valueless.get(var, ()) if valueless else ()
+ if not covered or _sees_value_from_before(reads.get(var, ()), s, e, declaring, bare):
+ params.append(var)
+
+ if d_lo < d_hi and _span_write_seen_after(reads.get(var, ()), s, e):
+ returns.append(var)
+ return tuple(params), tuple(returns)
+
+
+def _sees_value_from_before(
+ var_reads: list[tuple[int, frozenset[int]]],
+ s: int,
+ e: int,
+ declaring: Container[int],
+ bare: Container[int],
+) -> bool:
+ """Some read in ``[s, e]`` may observe a valued definition from before the span."""
+ return any(
+ s <= ln <= e and ln not in declaring and any(d < s and d not in bare for d in seen)
+ for ln, seen in var_reads
+ )
+
+
+def _span_write_seen_after(var_reads: list[tuple[int, frozenset[int]]], s: int, e: int) -> bool:
+ """Some read after line *e* may observe a write inside ``[s, e]``."""
+ return any(ln > e and any(s <= d <= e for d in seen) for ln, seen in var_reads)
+
+
+def _hoisted_bindings(
+ def_lines: dict[str, list[int]],
+ use_lines: dict[str, list[int]],
+) -> list[tuple[int, int]]:
+ """``(first_def, first_use)`` for names the function reads before defining them.
+
+ Such a read works only by hoisting (a JS/TS ``function foo()``), and an OUT
+ cannot express a value needed before the span runs, so a span holding a
+ ``first_def`` whose ``first_use`` sits above it is refused. Computed once
+ per function; both line lists are sorted, so their heads are the earliest.
+ """
+ hoisted: list[tuple[int, int]] = []
+ for var, defs in def_lines.items():
+ uses = use_lines.get(var)
+ if defs and uses and uses[0] < defs[0]:
+ hoisted.append((defs[0], uses[0]))
+ return sorted(hoisted)
+
+
+def _declaration_lines(
+ def_use: FunctionDefUse, *, binding: bool = False
+) -> dict[str, frozenset[int]]:
+ """Per variable, the lines that declare it (a ``let`` / ``var`` / ``:=`` /
+ typed local, as the dialect marks with ``declared_at``, or an import).
+ With *binding*, the lines that create a new binding instead
+ (``declares``: multi-line declarators and loop binders included, a TS/JS
+ ``var`` not)."""
+ lines: dict[str, set[int]] = defaultdict(set)
+ for d in def_use.definitions:
+ if (d.declares if binding else d.declared_at is not None or d.imports):
+ lines[d.var].add(d.line)
+ return {var: frozenset(found) for var, found in lines.items()}
+
+
+def _declaration_escapes(
+ s: int,
+ e: int,
+ returns: tuple[str, ...],
+ decl_lines: dict[str, frozenset[int]],
+ def_lines: dict[str, list[int]],
+ use_lines: dict[str, list[int]],
+ declared_first: dict[str, frozenset[int]],
+) -> bool:
+ """True when the span declares a name the code after it still refers to.
+
+ A returned name is fine: the caller declares it from the helper's result.
+ Otherwise the declaration leaves with the span and the caller's next
+ reference names nothing, even when that reference is a plain assignment
+ (Go ``var out T`` in the span, ``out = x`` after it, which is why liveness
+ did not make it an OUT). A later reference that declares the name afresh
+ (``y, err := g()``, a second ``for i := ...``, whose reads on that line
+ follow the declaration per *declared_first*) needs nothing from the span.
+ Block scopes are not modelled, so a same-named variable of an outer scope
+ read after the span also refuses: a missed span, never a broken one.
+ """
+ return any(
+ var not in returns
+ and any(s <= ln <= e for ln in declared)
+ and _needed_after(var, e, declared, def_lines, use_lines, declared_first)
+ for var, declared in decl_lines.items()
+ )
+
+
+def _bindings_escape(
+ s: int,
+ e: int,
+ returns: tuple[str, ...],
+ decl_lines: dict[str, frozenset[int]],
+ def_lines: dict[str, list[int]],
+ use_lines: dict[str, list[int]],
+ declared_first: dict[str, frozenset[int]],
+ outer: dict[str, frozenset[int]],
+) -> bool:
+ """A binding the span cannot take along: a declaration the code after it
+ still needs (:func:`_declaration_escapes`), or a ``global`` / ``nonlocal``
+ write whose declaration stays behind (:func:`_outer_write_escapes`)."""
+ return _declaration_escapes(
+ s, e, returns, decl_lines, def_lines, use_lines, declared_first
+ ) or _outer_write_escapes(s, e, outer, def_lines)
+
+
+def _outer_write_escapes(
+ s: int, e: int, outer: dict[str, frozenset[int]], def_lines: dict[str, list[int]]
+) -> bool:
+ """True when the span writes a ``global`` / ``nonlocal`` name it cannot
+ take its declaration along for: one declared outside the span, or also
+ written outside it. In a helper without the declaration the write binds a
+ local and the module value never changes."""
+ for var, declared in outer.items():
+ writes = def_lines.get(var, ())
+ if not any(s <= ln <= e for ln in writes):
+ continue
+ if any(not s <= ln <= e for ln in (*declared, *writes)):
+ return True
+ return False
+
+
+def _needed_after(
+ var: str,
+ e: int,
+ declared: frozenset[int],
+ def_lines: dict[str, list[int]],
+ use_lines: dict[str, list[int]],
+ declared_first: dict[str, frozenset[int]],
+) -> bool:
+ """True when the first reference to *var* after line *e* relies on an
+ earlier declaration: it is a plain use or assignment, not a fresh one."""
+ uses = use_lines.get(var, ())
+ after = [ln for ln in (*def_lines.get(var, ()), *uses) if ln > e]
+ if not after:
+ return False
+ first = min(after)
+ if first not in declared:
+ return True
+ return first in uses and first not in declared_first.get(var, ())
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice_nodes.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice_nodes.py
new file mode 100644
index 000000000..56b510046
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice_nodes.py
@@ -0,0 +1,182 @@
+"""Tree walkers the Extract Method slicer reads statement structure with."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from tree_sitter import Node
+
+ from ..complexity.languages import LanguageNodeMap
+
+
+def _function_lines(fn_node: Node) -> list[str]:
+ """Source rows indexed by absolute row, rebuilt from the function's own text.
+
+ Lets the walker's NLOC rule (``_code_line_numbers``) run here without the
+ file's bytes; rows above the function are never read, so they stay empty.
+ """
+ text = (fn_node.text or b"").decode("utf-8", errors="replace")
+ return [""] * fn_node.start_point[0] + text.splitlines()
+
+
+def _holds_a_named_nested_function(span: list[Node], lmap: LanguageNodeMap) -> bool:
+ """True when the span contains a nested function that binds its own name.
+
+ Def/use does not descend into nested scopes, so a sibling closure calling
+ that helper is invisible; lifting the declaration would strand the call.
+ An anonymous lambda bound to a local is a value and follows the IN/OUT rules.
+ """
+ kinds = lmap.function_kinds
+ if not kinds:
+ return False
+ for st in span:
+ stack = [st]
+ while stack:
+ node = stack.pop()
+ if node.type in kinds and node.child_by_field_name("name") is not None:
+ return True
+ stack.extend(node.children)
+ return False
+
+
+def _receiver_root(node: Node) -> str | None:
+ """The base name a call is made on: ``entries`` in ``entries.remove(x)``.
+
+ Only the root counts. Matching any name in the callee read
+ ``existing.pages.push(page)`` as touching the ``pages`` the loop iterates,
+ when the receiver is ``existing`` and the collision is in a member name.
+ """
+ cur = node
+ while True:
+ nxt = None
+ for field in ("object", "operand", "value", "argument", "function"):
+ child = cur.child_by_field_name(field)
+ if child is not None:
+ nxt = child
+ break
+ if nxt is None:
+ break
+ cur = nxt
+ if not cur.children and cur.type.endswith("identifier"):
+ text = cur.text
+ return text.decode("utf-8", "replace") if text else None
+ return None
+
+
+# Fields a loop header binds through. Read structurally: a line-range test
+# cannot tell the header's binder from a variable reassigned on the body's
+# first line, and getting that wrong drops the iterated collection out of the
+# carried set, which is exactly the name the mutation check exists to catch.
+_BINDER_FIELDS = ("left", "pattern", "declarator", "name")
+
+
+def _loop_header_names(loop: Node, lmap: LanguageNodeMap) -> set[str]:
+ """Names the loop header reads, minus the ones it binds (the loop variable)."""
+ body = loop.child_by_field_name("body")
+ names: set[str] = set()
+ # The binder is a field of the loop itself in most grammars, and of a
+ # header clause it wraps in Go's ``range``.
+ bound: set[str] = _binder_names(loop)
+ for child in loop.children:
+ if body is not None and child.id == body.id:
+ continue
+ names |= _identifiers(child)
+ bound |= _binder_names(child)
+ return names - bound
+
+
+def _binder_names(node: Node) -> set[str]:
+ """Identifiers in the binding position of *node* or of a header clause it
+ wraps (Go nests its range clause inside the ``for``)."""
+ out: set[str] = set()
+ for field in _BINDER_FIELDS:
+ target = node.child_by_field_name(field)
+ if target is not None:
+ out |= _identifiers(target)
+ if not out:
+ for child in node.named_children:
+ for field in _BINDER_FIELDS:
+ target = child.child_by_field_name(field)
+ if target is not None:
+ out |= _identifiers(target)
+ return out
+
+
+def _identifiers(node: Node) -> set[str]:
+ out: set[str] = set()
+ stack = [node]
+ while stack:
+ cur = stack.pop()
+ if not cur.children and cur.type.endswith("identifier"):
+ text = cur.text
+ if text:
+ out.add(text.decode("utf-8", "replace"))
+ stack.extend(cur.children)
+ return out
+
+
+def _descend(root: Node, kinds: frozenset[str], skip: frozenset[str]) -> list[Node]:
+ found: list[Node] = []
+ stack = [root]
+ while stack:
+ cur = stack.pop()
+ if cur.type in kinds:
+ found.append(cur)
+ for child in cur.children:
+ if child.type in skip:
+ continue
+ stack.append(child)
+ return found
+
+
+def _unwrap_container(node: Node, block_kinds: frozenset[str]) -> Node:
+ """Descend through a single nested statement-container (Go's
+ ``block`` -> ``statement_list``) to the node whose named children are the
+ actual statements; returns *node* unchanged when it is already that node."""
+ cur = node
+ while True:
+ named = cur.named_children
+ if len(named) == 1 and named[0].type in block_kinds:
+ cur = named[0]
+ else:
+ return cur
+
+
+def all_blocks(
+ fn_node: Node,
+ block_kinds: frozenset[str],
+ scope_kinds: frozenset[str],
+ loop_kinds: frozenset[str],
+) -> list[tuple[Node, Node | None]]:
+ """Every statement container in the function (body + nested), excluding the
+ bodies of nested functions / lambdas, each paired with the innermost loop
+ enclosing it (``None`` at loop-free depth)."""
+ body = fn_node.child_by_field_name("body")
+ if body is None:
+ return []
+ blocks: list[tuple[Node, Node | None]] = []
+ stack: list[tuple[Node, Node | None]] = [(body, None)]
+ while stack:
+ node, loop = stack.pop()
+ if node.type in block_kinds:
+ blocks.append((node, loop))
+ is_loop = node.type in loop_kinds
+ body = node.child_by_field_name("body") if is_loop else None
+ for child in node.children:
+ if child.type in scope_kinds:
+ continue
+ # Only the loop's body repeats. A ``for ... else`` clause runs once.
+ in_loop = node if (is_loop and body is not None and child.id == body.id) else loop
+ stack.append((child, in_loop))
+ return blocks
+
+
+def _exit_macros(lmap: LanguageNodeMap) -> tuple[frozenset[str], frozenset[str]]:
+ """The macro node kinds and the macro names that exit the function."""
+ return lmap.exit_macro_kinds, lmap.exit_macro_names
+
+
+def _awaits(lmap: LanguageNodeMap) -> tuple[frozenset[str], frozenset[str]]:
+ """The await tokens, and the nodes that own the awaits inside them."""
+ return lmap.await_kinds, lmap.await_scope_kinds
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice_receiver.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice_receiver.py
new file mode 100644
index 000000000..ccbff0d05
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice_receiver.py
@@ -0,0 +1,252 @@
+"""What a span or a whole function does with its receiver (``self`` / ``this``)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from ..complexity.ast_utils import member_object, self_member_name
+from ..complexity.body_facts import BodyTally, FactKinds, is_exit
+from .dialects.base import SUPER, mentions_receiver
+from .slice_nodes import _exit_macros
+
+if TYPE_CHECKING:
+ from tree_sitter import Node
+
+ from ..complexity.languages import LanguageNodeMap
+ from .defuse import FunctionDefUse
+ from .dialects.base import Receiver
+ from .slice import _Prefix
+
+
+@dataclass(frozen=True)
+class FunctionFacts:
+ """What a whole function does, counted on the complexity walk's visits.
+
+ ``awaits``: it suspends outside any nested scope. ``is_generator``: it
+ yields. ``uses_receiver`` / ``receiver_assigns`` as on :class:`Extraction`,
+ for the whole body; ``receiver_assigns`` is None where an implicit receiver
+ (Java, C++) can write fields by bare name, which only def/use can tell.
+ ``early_exits``: returns and raises other than a final one.
+ """
+
+ awaits: bool
+ is_generator: bool
+ uses_receiver: bool | None
+ receiver_assigns: tuple[str, ...] | None
+ early_exits: int
+
+
+class _ReceiverSink:
+ """``BodyTally.on_receiver``: whether the body names its receiver, and
+ which of its fields it assigns."""
+
+ __slots__ = ("assigns", "receiver", "uses")
+
+ def __init__(self, receiver: Receiver) -> None:
+ self.receiver = receiver
+ self.uses = False
+ self.assigns: set[str] = set()
+
+ def __call__(self, node: Node) -> None:
+ self.uses = _receiver_ref(node, self.receiver, self.assigns) or self.uses
+
+
+def body_tally(fn_node: Node, lmap: LanguageNodeMap, receiver: Receiver | None) -> BodyTally:
+ """An empty tally for *fn_node*, for the complexity walk to fill. The
+ receiver is watched only when the function's text names it."""
+ watched = bool(
+ receiver is not None and receiver.names and mentions_receiver(fn_node, receiver.names)
+ )
+ sink = _ReceiverSink(receiver) if watched and receiver is not None else None
+ return BodyTally(_function_kinds(lmap, sink), sink)
+
+
+# Built once per language map and receiver shape: the walk asks per function.
+_FUNCTION_KINDS: dict[tuple[int, frozenset[str]], FactKinds] = {}
+
+
+def _function_kinds(lmap: LanguageNodeMap, sink: _ReceiverSink | None) -> FactKinds:
+ receiver = _receiver_kinds(sink.receiver) if sink else frozenset()
+ key = (id(lmap), receiver)
+ kinds = _FUNCTION_KINDS.get(key)
+ if kinds is None:
+ kinds = _FUNCTION_KINDS[key] = FactKinds(
+ exits=lmap.return_kinds | lmap.raise_kinds,
+ exit_macros=_exit_macros(lmap),
+ yields=lmap.yield_kinds,
+ awaits=lmap.await_kinds,
+ await_scopes=lmap.await_scope_kinds,
+ lambdas=lmap.lambda_kinds,
+ receiver=receiver,
+ )
+ return kinds
+
+
+def _receiver_kinds(receiver: Receiver) -> frozenset[str]:
+ """What :func:`_receiver_ref` can answer for: a write, or a receiver leaf."""
+ return receiver.write_kinds | receiver.names | {SUPER, "identifier"}
+
+
+def function_facts(
+ fn_node: Node, lmap: LanguageNodeMap, receiver: Receiver | None, tally: BodyTally
+) -> FunctionFacts:
+ """:class:`FunctionFacts` from a filled :func:`body_tally`."""
+ body = fn_node.child_by_field_name("body") or fn_node
+ sink = tally.on_receiver if isinstance(tally.on_receiver, _ReceiverSink) else None
+ uses, assigns = _whole_receiver_facts(
+ receiver, sink.uses if sink else False, frozenset(sink.assigns) if sink else frozenset()
+ )
+ return FunctionFacts(
+ awaits=tally.awaits,
+ is_generator=tally.yields > 0,
+ uses_receiver=uses,
+ receiver_assigns=assigns,
+ early_exits=max(0, tally.exits - _ends_in_exit(body, lmap, tally.kinds)),
+ )
+
+
+def _ends_in_exit(body: Node, lmap: LanguageNodeMap, kinds: FactKinds) -> bool:
+ """Whether a block body's last statement leaves the function, so that exit
+ is not an early one. The statement is the exit (Python ``return x``, TS
+ ``throw e;``) or wraps it as its first named child (Rust ``return x;`` and
+ ``bail!(..);`` are expression statements). An expression body has no
+ statements, so its exits are early by definition and none are subtracted."""
+ if body.type not in lmap.block_kinds or not body.named_children:
+ return False
+ tail = body.named_children[-1]
+ if is_exit(tail, kinds):
+ return True
+ return bool(tail.named_children) and is_exit(tail.named_children[0], kinds)
+
+
+def _whole_receiver_facts(
+ receiver: Receiver | None, uses: bool, assigned: frozenset[str]
+) -> tuple[bool | None, tuple[str, ...] | None]:
+ if receiver is None:
+ return None, None
+ if not receiver.names:
+ return False, ()
+ if receiver.implicit:
+ return (True if uses else None), None
+ return uses, None if _UNREAD_TARGET in assigned else tuple(sorted(assigned))
+
+
+def _free_write_lines(def_use: FunctionDefUse) -> list[int]:
+ """Sorted lines writing a name the function neither declares nor takes
+ as a parameter: in Java or C++ that is a field (or a C++ global) written
+ without ``this``, so the span's receiver writes are not all known."""
+ params = {p.name for p in def_use.params}
+ declared = {d.var for d in def_use.definitions if d.declares or d.declared_at is not None}
+ return sorted(
+ d.line for d in def_use.definitions if d.var not in params and d.var not in declared
+ )
+
+
+def _receiver_facts(
+ receiver: Receiver | None, pre: _Prefix, i: int, j: int, free_writes: int
+) -> tuple[bool | None, tuple[str, ...] | None]:
+ """``uses_receiver`` / ``receiver_assigns`` for the span over statements
+ ``i..j`` (see :class:`Extraction`)."""
+ if receiver is None:
+ return None, None
+ if not receiver.names:
+ return False, ()
+ named = pre.receiver[j + 1] > pre.receiver[i]
+ found = frozenset().union(*pre.assigns[i : j + 1])
+ fields = None if _UNREAD_TARGET in found else tuple(sorted(found))
+ if receiver.implicit:
+ return (True if named else None), (None if free_writes else fields)
+ return named, fields
+
+
+# Index access on a receiver field (``self.cache[k] = v``) assigns into it.
+_INDEX_KINDS = frozenset({"subscript", "subscript_expression", "index_expression", "array_access"})
+# Unpacking targets whose named children are targets themselves
+# (``a, self.b = ...``, ``[this.x, y] = ...``).
+_UNPACK_KINDS = frozenset(
+ {
+ "pattern_list",
+ "tuple_pattern",
+ "list_pattern",
+ "expression_list",
+ "tuple",
+ "list",
+ "array_pattern",
+ "object_pattern",
+ "parenthesized_expression",
+ "list_splat_pattern",
+ "rest_pattern",
+ }
+)
+# Marks a target rooted at the receiver whose shape is not read, so the
+# span's assigned fields are unknown.
+_UNREAD_TARGET = ""
+
+
+def _receiver_ref(node: Node, receiver: Receiver, assigns: set[str]) -> bool:
+ """True when *node* names the receiver; an assignment to one of its
+ fields adds the field's name to *assigns* (the receiver leaf under it is
+ visited next)."""
+ if node.type in receiver.write_kinds:
+ target = (
+ node.child_by_field_name("left")
+ or node.child_by_field_name("argument")
+ or (node.named_children[0] if node.named_children else None)
+ )
+ if target is not None:
+ _collect_assigned(target, receiver, assigns)
+ return False
+ return _is_receiver(node, receiver)
+
+
+def _collect_assigned(target: Node, receiver: Receiver, assigns: set[str]) -> None:
+ """The receiver fields one assignment target writes, unpacking included;
+ a target of another shape that mentions the receiver makes them unknown."""
+ if target.type in _UNPACK_KINDS:
+ for each in target.named_children:
+ _collect_assigned(each, receiver, assigns)
+ return
+ if target.type in ("assignment_pattern", "object_assignment_pattern"): # ``[this.a = 1]``
+ left = target.child_by_field_name("left")
+ if left is not None:
+ _collect_assigned(left, receiver, assigns)
+ return
+ if target.type == "pair_pattern": # ``{key: this.a}``
+ value = target.child_by_field_name("value")
+ if value is not None:
+ _collect_assigned(value, receiver, assigns)
+ return
+ field = _receiver_field(target, receiver)
+ if field:
+ assigns.add(field)
+ elif target.type not in receiver.access_kinds | _INDEX_KINDS and mentions_receiver(
+ target, receiver.names
+ ):
+ assigns.add(_UNREAD_TARGET)
+
+
+def _is_receiver(node: Node, receiver: Receiver) -> bool:
+ """A leaf naming the receiver: a ``this`` / ``self`` / ``super`` token,
+ or an identifier spelling a named receiver (Python ``self``, Go ``s``)
+ or ``super`` (Python's no-argument ``super()``)."""
+ if node.children or not node.text:
+ return False
+ if node.type in receiver.names or node.type == SUPER:
+ return True
+ if node.type != "identifier":
+ return False
+ text = node.text.decode("utf-8", "replace")
+ return text in receiver.names or text == SUPER
+
+
+def _receiver_field(target: Node, receiver: Receiver) -> str | None:
+ """``x`` for an assignment target ``self.x``, ``self.x.y`` or ``self.x[k]``."""
+ cur: Node | None = target
+ while cur is not None and (cur.type in receiver.access_kinds or cur.type in _INDEX_KINDS):
+ if cur.type in receiver.access_kinds:
+ field = self_member_name(cur, receiver.names)
+ if field is not None:
+ return field
+ cur = member_object(cur)
+ return None
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice_state.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice_state.py
new file mode 100644
index 000000000..b82602cf3
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice_state.py
@@ -0,0 +1,240 @@
+"""State a span shares with code it does not hold: closure-shared locals and loop-carried values."""
+
+from __future__ import annotations
+
+from bisect import bisect_left, bisect_right
+from collections import defaultdict
+from typing import TYPE_CHECKING, NamedTuple
+
+from .slice_liveness import _declaration_lines
+from .slice_nodes import _descend, _loop_header_names, _receiver_root
+
+if TYPE_CHECKING:
+ from tree_sitter import Node
+
+ from ..complexity.languages import LanguageNodeMap
+ from .analyze import FunctionAnalysis
+ from .defuse import FunctionDefUse
+
+
+class _SharedState(NamedTuple):
+ """Per variable a closure shares with the function, sorted line lists:
+ the closures' reads and writes (``Captured.shared`` / ``.writes``), the
+ function's own defs, all its references, and its binding declarations."""
+
+ reads: dict[str, list[int]]
+ writes: dict[str, list[int]]
+ defs: dict[str, list[int]]
+ refs: dict[str, list[int]]
+ binds: dict[str, frozenset[int]]
+
+
+def _closure_state(
+ def_use: FunctionDefUse,
+ def_lines: dict[str, list[int]],
+ use_lines: dict[str, list[int]],
+) -> _SharedState | None:
+ """The closure-shared locals of a function, or None when it has none
+ (then the gate costs nothing per span)."""
+ reads: dict[str, list[int]] = defaultdict(list)
+ writes: dict[str, list[int]] = defaultdict(list)
+ for out, occurrences in (
+ (reads, def_use.captured.shared),
+ (writes, def_use.captured.writes),
+ ):
+ for occ in occurrences:
+ if occ.name in def_lines: # a name this function binds
+ out[occ.name].append(occ.line)
+ if not reads and not writes:
+ return None
+ names = set(reads) | set(writes)
+ return _SharedState(
+ reads={var: sorted(lines) for var, lines in reads.items()},
+ writes={var: sorted(lines) for var, lines in writes.items()},
+ defs={var: def_lines.get(var, []) for var in names},
+ refs={var: sorted((*def_lines.get(var, ()), *use_lines.get(var, ()))) for var in names},
+ binds=_declaration_lines(def_use, binding=True),
+ )
+
+
+def _count_in(lines: list[int], s: int, e: int) -> int:
+ return bisect_right(lines, e) - bisect_left(lines, s)
+
+
+def _closure_state_crosses(
+ st: _SharedState, span: list[Node], s: int, e: int, block: Node, lmap: LanguageNodeMap
+) -> bool:
+ """True when a local a closure shares crosses the span boundary.
+
+ A closure runs when it is called, not where it is written, so line
+ liveness cannot place its reads and writes; an IN/OUT signature copies a
+ value where the closure needs the variable itself. Refused:
+
+ - a closure in the span writes a local anything outside the span refers
+ to (code, or a closure there): the lifted closure writes the helper's
+ copy. A counter declared in the span with its ``inc`` closure is
+ refused this way when the count is read after the span;
+ - a closure outside the span writes a local the span refers to: the span
+ reads or writes a stale copy;
+ - the span writes a local that a closure written above it reads: the
+ closure runs later and sees the old value.
+
+ The last two are waived when the span's name is a binding of its own
+ (:func:`_span_binds`, :func:`_span_only_rebinds`): the closure's variable
+ is then another one.
+ """
+ for var in st.reads.keys() | st.writes.keys():
+ writes = st.writes.get(var, [])
+ reads = st.reads.get(var, [])
+ refs = st.refs[var]
+ w_in = _count_in(writes, s, e)
+ if w_in and (
+ w_in < len(writes)
+ or _count_in(reads, s, e) < len(reads)
+ or _count_in(refs, s, e) < len(refs)
+ ):
+ return True
+ stale = w_in < len(writes) and _count_in(refs, s, e) > 0
+ if stale and not _span_binds(st, var, span, block, lmap):
+ return True
+ overwritten = bool(reads) and reads[0] < s and _count_in(st.defs[var], s, e) > 0
+ if overwritten and not _span_only_rebinds(st, var, s, e, block, lmap):
+ return True
+ return False
+
+
+def _declares_in(node: Node, lines: frozenset[int], lmap: LanguageNodeMap) -> bool:
+ """True when *node* is a declaration statement holding one of *lines*."""
+ lo, hi = node.start_point[0] + 1, node.end_point[0] + 1
+ return node.type in lmap.local_decl_kinds and any(lo <= ln <= hi for ln in lines)
+
+
+def _declared_above(binds: frozenset[int], block: Node, s: int, lmap: LanguageNodeMap) -> bool:
+ """True when a statement of *block* above the span already declares the
+ name: a re-declaration in the same scope (Go's ``v, err := h()``) is an
+ assignment to it, not a new binding."""
+ return any(
+ node.end_point[0] + 1 < s and _declares_in(node, binds, lmap)
+ for node in block.named_children
+ )
+
+
+def _span_binds(
+ st: _SharedState, var: str, span: list[Node], block: Node, lmap: LanguageNodeMap
+) -> bool:
+ """True when one of the span's own top-level statements declares a new
+ binding of *var*, so every reference to it in the span is to that one.
+
+ Top-level only: a declaration in a nested block shadows just that block.
+ """
+ binds = st.binds.get(var, frozenset())
+ s = span[0].start_point[0] + 1
+ return (
+ bool(binds)
+ and not _declared_above(binds, block, s, lmap)
+ and any(_declares_in(node, binds, lmap) for node in span)
+ )
+
+
+def _span_only_rebinds(
+ st: _SharedState, var: str, s: int, e: int, block: Node, lmap: LanguageNodeMap
+) -> bool:
+ """True when every write the span makes to *var* creates a new binding
+ (a declaration or a loop binder, at any depth), so none of them reaches
+ the variable a closure outside the span reads."""
+ binds = st.binds.get(var, frozenset())
+ defs = st.defs[var][bisect_left(st.defs[var], s) : bisect_right(st.defs[var], e)]
+ return all(ln in binds for ln in defs) and not _declared_above(binds, block, s, lmap)
+
+
+def _rebinding_defs_dropped(
+ analysis: FunctionAnalysis, def_lines: dict[str, list[int]], lmap: LanguageNodeMap
+) -> dict[str, list[int]]:
+ """*def_lines* without the names exception handlers bind on their head
+ line (``except E as e:``, ``catch (err)``). Such a binding never reaches
+ the next iteration: Python unbinds it at the handler's end and a catch
+ parameter is scoped to its block, so a read of the name elsewhere in the
+ loop is another handler's own."""
+ heads = _catch_heads(analysis, lmap)
+ if not heads:
+ return def_lines
+ bound: set[tuple[str, int]] = set()
+ for head in heads:
+ bound |= _head_bindings(head, analysis.def_use, lmap)
+ return {var: [ln for ln in lines if (var, ln) not in bound] for var, lines in def_lines.items()}
+
+
+def _catch_heads(analysis: FunctionAnalysis, lmap: LanguageNodeMap) -> list[Node]:
+ return [
+ st.node
+ for block in analysis.cfg.blocks
+ for st in block.statements
+ if st.is_head and st.kind in lmap.catch_kinds and st.node is not None
+ ]
+
+
+def _head_bindings(
+ head: Node, def_use: FunctionDefUse, lmap: LanguageNodeMap
+) -> set[tuple[str, int]]:
+ # Only the clause's own binder: before its body, so a one-line body's
+ # write on the same line still counts.
+ body = next((c for c in head.named_children if c.type in lmap.block_kinds), None)
+ end = body.start_point if body is not None else head.end_point
+ return {
+ (d.var, d.line)
+ for d in def_use.definitions
+ if head.start_point <= (d.line - 1, d.column) < end
+ }
+
+
+def _loop_carry_free(
+ span: list[Node],
+ loop: Node,
+ s: int,
+ e: int,
+ def_lines: dict[str, list[int]],
+ use_lines: dict[str, list[int]],
+ lmap: LanguageNodeMap,
+) -> bool:
+ """True when a span nested in *loop* carries no state between iterations.
+
+ Refused: a variable the span writes that the loop reads with no in-span
+ write before the read (the read takes the previous iteration's value, even
+ when it sits inside the span above the write), and a call on a name the
+ loop header reads (``entries.remove(entry)`` in ``for entry in entries``).
+ """
+ if _carries_a_write(loop.start_point[0] + 1, s, e, def_lines, use_lines):
+ return False
+ carried = _loop_header_names(loop, lmap)
+ return not carried or not _mutates_carried(span, carried, lmap)
+
+
+def _carries_a_write(
+ loop_start: int,
+ s: int,
+ e: int,
+ def_lines: dict[str, list[int]],
+ use_lines: dict[str, list[int]],
+) -> bool:
+ for var, lines in def_lines.items():
+ in_span_writes = [ln for ln in lines if s <= ln <= e]
+ if not in_span_writes:
+ continue
+ for read in use_lines.get(var, []):
+ if loop_start <= read <= e and not any(write < read for write in in_span_writes):
+ return True
+ return False
+
+
+def _mutates_carried(span: list[Node], carried: set[str], lmap: LanguageNodeMap) -> bool:
+ scope_kinds = lmap.function_kinds | lmap.lambda_kinds
+ for st in span:
+ for call in _descend(st, lmap.call_kinds, scope_kinds):
+ callee = call.child_by_field_name("function")
+ if callee is None:
+ named = call.named_children
+ callee = named[0] if named else None
+ root = _receiver_root(callee) if callee is not None else None
+ if root is not None and root in carried:
+ return True
+ return False
diff --git a/packages/core/src/repowise/core/analysis/health/dataflow/slice_writes.py b/packages/core/src/repowise/core/analysis/health/dataflow/slice_writes.py
new file mode 100644
index 000000000..31abd6d4b
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/dataflow/slice_writes.py
@@ -0,0 +1,143 @@
+"""Structural proof that a span writes each OUT variable on every path through it."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .slice_nodes import _identifiers, _unwrap_container
+
+if TYPE_CHECKING:
+ from tree_sitter import Node
+
+ from ..complexity.languages import LanguageNodeMap
+
+
+def _outs_definitely_assigned(
+ span: list[Node],
+ returns: tuple[str, ...],
+ def_lines: dict[str, list[int]],
+ lmap: LanguageNodeMap,
+) -> bool:
+ """True when every OUT variable is written on *every* path through the span.
+
+ A conditionally written OUT is the unsound case line-based liveness cannot
+ see: the helper returns the unwritten value on the paths that skip the
+ write, and the caller's assignment then clobbers the live one. The proof
+ needs branch structure, so it runs structurally here and the span is
+ refused whenever the proof does not go through.
+ """
+ return all(_stmts_assign(span, var, def_lines.get(var, []), lmap) for var in returns)
+
+
+def _stmts_assign(stmts: list[Node], var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:
+ """True when *stmts* writes *var* on every path through them."""
+ return any(_stmt_assigns(st, var, defs, lmap) for st in stmts)
+
+
+def _stmt_assigns(st: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:
+ """True only when *st* is proved to write *var* however it is entered.
+
+ Positive proof only: a statement kind this cannot classify returns False.
+ Defaulting to True would let any statement sharing a line with a
+ conditional write stand in as the proof.
+ """
+ lo = st.start_point[0] + 1
+ hi = st.end_point[0] + 1
+ if not any(lo <= d <= hi for d in defs):
+ return False
+ if _is_conditional(st, lmap):
+ return _conditional_assigns(st, var, defs, lmap)
+ if st.type in lmap.block_kinds:
+ container = _unwrap_container(st, lmap.block_kinds)
+ return _stmts_assign(container.named_children, var, defs, lmap)
+ if st.type in lmap.with_kinds or st.type in lmap.statement_wrapper_kinds:
+ return _stmts_assign(list(st.named_children), var, defs, lmap)
+ return _unconditional_write(st, var, lmap)
+
+
+def _unconditional_write(st: Node, var: str, lmap: LanguageNodeMap) -> bool:
+ """True when *st* contains a write to *var* no branch or loop guards.
+
+ The walk stops at any control container (a loop body may run zero times, a
+ branch arm may not be taken, a catch arm may not fire) and at nested
+ scopes, so only writes on the statement's own straight-line path count.
+ """
+ write_kinds = lmap.assignment_kinds | lmap.augmented_assign_kinds | lmap.local_decl_kinds
+ if not write_kinds:
+ return False
+ barriers = (
+ lmap.loop_kinds
+ | lmap.try_kinds
+ | lmap.catch_kinds
+ | lmap.switch_kinds
+ | lmap.case_kinds
+ | lmap.branch_kinds
+ | lmap.if_kinds
+ | lmap.function_kinds
+ | lmap.lambda_kinds
+ )
+ if st.type in barriers:
+ return False
+ stack = [st]
+ while stack:
+ node = stack.pop()
+ if node.id != st.id and node.type in barriers:
+ continue
+ if node.type in write_kinds and _writes_var(node, var):
+ return True
+ stack.extend(node.children)
+ return False
+
+
+def _writes_var(node: Node, var: str) -> bool:
+ """True when *var* is on the binding side of this write node."""
+ for field in ("left", "pattern", "declarator", "name"):
+ target = node.child_by_field_name(field)
+ if target is not None:
+ return var in _identifiers(target)
+ return var in _identifiers(node)
+
+
+def _is_conditional(node: Node, lmap: LanguageNodeMap) -> bool:
+ return node.type in lmap.if_kinds or (
+ node.type in lmap.branch_kinds and node.child_by_field_name("condition") is not None
+ )
+
+
+def _conditional_assigns(node: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:
+ """True when a conditional chain is exhaustive and every arm writes *var*.
+
+ Two chain shapes. Python flattens ``elif``/``else`` into repeated
+ ``alternative`` fields on one ``if``, so the terminal else is a sibling of
+ the chained arms. The C family nests instead: the single ``alternative`` is
+ another ``if``, whose own else ends the chain. Both must be exhaustive.
+ """
+ consequence = node.child_by_field_name("consequence") or node.child_by_field_name("body")
+ if consequence is None:
+ return False
+ alternatives = list(node.children_by_field_name("alternative"))
+ if not alternatives:
+ return False # no else at all: some path skips the write
+ if not _branch_assigns(consequence, var, defs, lmap):
+ return False
+ for alt in alternatives:
+ if _is_conditional(alt, lmap):
+ # Nested chain (C family): it carries the chain's terminal else.
+ if len(alternatives) == 1:
+ return _conditional_assigns(alt, var, defs, lmap)
+ # Flattened chain (Python's ``elif``): the else is a sibling below.
+ arm = alt.child_by_field_name("consequence") or alt.child_by_field_name("body")
+ if arm is None or not _branch_assigns(arm, var, defs, lmap):
+ return False
+ elif not _branch_assigns(alt, var, defs, lmap):
+ return False
+ return any(not _is_conditional(alt, lmap) for alt in alternatives)
+
+
+def _branch_assigns(node: Node, var: str, defs: list[int], lmap: LanguageNodeMap) -> bool:
+ if _is_conditional(node, lmap):
+ return _conditional_assigns(node, var, defs, lmap)
+ if node.type in lmap.block_kinds:
+ container = _unwrap_container(node, lmap.block_kinds)
+ return _stmts_assign(container.named_children, var, defs, lmap)
+ return _stmts_assign(list(node.named_children), var, defs, lmap)
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
index 669926a32..47375dd8f 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
@@ -1,11 +1,12 @@
"""Python ``PerfDialect``.
-Extracted verbatim from the original ``_py_sink_kind`` (``io_boundaries.py``)
-and the Python branches of the walker (``_blocking_sync_api`` / ``_is_constant_for``
-/ the ``_PY_STRING_KINDS`` string-concat predicate). The 6a refactor changes
-zero Python behavior — the defect golden + perf suite lock that.
+Vocabularies live in :mod:`.python_lexicon`, syntax helpers in
+:mod:`.python_nodes`, loop sources in :mod:`.python_sources` and batched forms
+in :mod:`.python_batch`. Their names are re-exported here.
"""
+# ruff: noqa: F401
+
from __future__ import annotations
import builtins
@@ -16,367 +17,88 @@ from .. import loop_lexicon as lex
from ..loop_facts import BatchForm, LoopSource
from .base import BasePerfDialect
from .iterables import IterableSources
-
-if TYPE_CHECKING:
- from tree_sitter import Node
-
- from ..loop_facts import LoopIterates, LoopMagnitude, SinkProbe
-
- Source = tuple[LoopIterates, LoopMagnitude | None]
-
-# Network round-trip verbs (shared across languages that name HTTP methods).
-HTTP_VERBS: frozenset[str] = frozenset(
- {"get", "post", "put", "delete", "patch", "head", "options", "request", "send", "stream"}
-)
-
-# Python DBAPI / SQLAlchemy execution sinks, split into three strata so the
-# ambiguous accessors can be gated harder than the unambiguous ones (see
-# ``sink_kind``).
-PY_DB_UNAMBIGUOUS: frozenset[str] = frozenset({"fetchone", "fetchall", "fetchmany"})
-# DB verbs other clients spell too: GitPython's ``repo.commit()``, an HTTP or
-# GraphQL SDK's ``request.execute()``, a builder's ``.scalar()``.
-PY_DB_NEEDS_EVIDENCE: frozenset[str] = frozenset(
- {"commit", "execute", "executemany", "scalars", "scalar", "scalar_one", "scalar_one_or_none"}
+from .python_batch import PythonBatchForms
+from .python_lexicon import (
+ _PY_AUG_ASSIGN_KINDS,
+ _PY_BINDING_PARENTS,
+ _PY_BINDING_WRAPPERS,
+ _PY_BUILTINS,
+ _PY_CONSTANT_RHS,
+ _PY_DEF_KINDS,
+ _PY_KEY_READS,
+ _PY_LIST_BUILTINS,
+ _PY_LIST_RHS_KINDS,
+ _PY_LISTINGS,
+ _PY_LOCK_METHODS,
+ _PY_MODEL_NAME_RE,
+ _PY_NONLIST_BUILTINS,
+ _PY_NONLIST_RHS_KINDS,
+ _PY_ONE_ROW,
+ _PY_PASS_THROUGH,
+ _PY_PER_KEY_LIMITS,
+ _PY_QUERY_CHAIN_METHODS,
+ _PY_READ_CAPS,
+ _PY_RECURSIVE_LISTINGS,
+ _PY_RESOURCE_CTORS,
+ _PY_RESOURCE_METHODS,
+ _PY_RESULT_PROJECTIONS,
+ _PY_SCOPES,
+ _PY_SEM_NAME_RE,
+ _PY_SQL_COMMENT_RE,
+ _PY_SQL_LIMIT_RE,
+ _PY_SQL_LITERAL_RE,
+ _PY_STATEMENT_BUILDERS,
+ _PY_STREAMS,
+ _PY_STRING_KINDS,
+ _PY_WRITES,
+ HTTP_VERBS,
+ PY_DB_AMBIGUOUS,
+ PY_DB_NEEDS_EVIDENCE,
+ PY_DB_UNAMBIGUOUS,
+ PY_FS_METHODS,
+ PY_OS_FS_METHODS,
+ PY_SHUTIL_FS_METHODS,
+ PY_SUBPROC_METHODS,
+ PY_UNBOUNDED_READ_BOUND_METHODS,
)
-# Without a db import, the call itself can still show it runs a query: a
-# PostgREST chain (``client.table("t")...execute()``) or a statement argument
-# (``session.execute(select(...))``, ``cur.execute("SELECT ...")``).
-_PY_QUERY_CHAIN_METHODS: frozenset[str] = frozenset({"table", "from_"})
-_PY_STATEMENT_BUILDERS: frozenset[str] = frozenset({"select", "insert", "update", "delete", "text"})
-_PY_SQL_LITERAL_RE = re.compile(
- r"^[rRuUbBfF]*[\"']+\s*(?:SELECT|INSERT|UPDATE|DELETE|WITH|CREATE|DROP|ALTER|PRAGMA)\b",
- re.IGNORECASE,
-)
-PY_DB_AMBIGUOUS: frozenset[str] = frozenset({"all", "first", "one", "one_or_none"})
-PY_SUBPROC_METHODS: frozenset[str] = frozenset(
- {"run", "call", "check_call", "check_output", "Popen"}
-)
-# Any of these anywhere in a read's call chain proves it is bounded, gating
-# ``unbounded_read_reduced_in_memory`` (analysis.health.perf.unbounded_reduction).
-PY_UNBOUNDED_READ_BOUND_METHODS: frozenset[str] = frozenset(
- {"limit", "range", "single", "maybe_single", "first", "count", "head", "aggregate"}
+from .python_nodes import (
+ _PY_SCALARS,
+ _bindings,
+ _binds,
+ _field_text,
+ _fixed_set,
+ _in_filter,
+ _literal_range,
+ _module_int_constant,
+ _scalar_literal,
+ _string_value,
+ _text,
+ _value_refs,
)
+from .python_sources import PythonLoopSources
-# Filesystem round-trips, in two strata for the same reason the DB verbs are
-# stratified: how much evidence the name alone carries.
-#
-# ``PY_FS_METHODS`` are matched on the method name alone, because in this
-# ecosystem only a ``pathlib.Path`` spells them — the same reasoning the TS
-# dialect applies to ``readFileSync``. That matters because the dominant real
-# shape binds the path to a local first (``src_path.read_text()``), so the
-# receiver is an ordinary variable and ``io_names`` cannot classify it; keying
-# on the module would miss nearly every genuine site. The Kotlin dialect
-# deliberately excludes its ``readText`` / ``readBytes`` homonyms because
-# kotlinx-io and Ktor mirror those names on in-memory buffers; Python has no
-# such homonym (``io.StringIO`` and every stream type spell it ``.read()``),
-# which is what makes the bare-name match safe here and not there.
-#
-# Metadata syscalls (``exists`` / ``is_file`` / ``is_dir`` / ``stat`` /
-# ``glob``) are deliberately absent. A stat in a loop is cheap and usually
-# correct, so flagging it is the obvious false-positive class — the same call
-# the C++ dialect makes when it excludes buffered stdio.
-PY_FS_METHODS: frozenset[str] = frozenset(
- {"read_text", "read_bytes", "write_text", "write_bytes"}
-)
-# ``os`` and ``shutil`` verbs, gated on the literal module root the way
-# ``subprocess`` is. ``os`` is far too broad to classify wholesale (``os.environ``
-# / ``os.getpid`` / ``os.path.join`` are not I/O), so it never goes through
-# ``io_names``; the root name plus an explicit verb set is the whole gate.
-#
-# CEILING: ``callee_root_name`` resolves the LEFTMOST identifier of the whole
-# callee chain, so ``os.path.join(a, b).verb()`` also arrives here as
-# ``root == "os"``. The verb set must therefore hold no name that a ``str`` /
-# ``list`` / ``dict`` also answers to. ``replace`` was the one that did, and it
-# was a live false positive on the canonical Windows path idiom
-# (``os.path.join(root, name).replace("\\", "/")`` — five instances across the
-# OSS corpus, each yielding a bogus ``io_in_loop`` AND a ``nested_loop_with_io``).
-# It is omitted at near-zero recall cost: ``os.replace`` is atomic ``os.rename``,
-# which is already listed. Upgrade path if this set ever needs a colliding verb:
-# thread the callee's dotted segment count into ``sink_kind`` and require
-# exactly two, which is the real invariant being approximated here.
-PY_OS_FS_METHODS: frozenset[str] = frozenset(
- {
- "walk",
- "listdir",
- "scandir",
- "remove",
- "unlink",
- "rename",
- "makedirs",
- "mkdir",
- "rmdir",
- "removedirs",
- }
-)
-PY_SHUTIL_FS_METHODS: frozenset[str] = frozenset(
- {
- "copy",
- "copy2",
- "copyfile",
- "copytree",
- "move",
- "rmtree",
- "make_archive",
- "unpack_archive",
- }
-)
-
-# Python string-literal node kinds (f-strings parse as ``string`` too).
-_PY_STRING_KINDS: frozenset[str] = frozenset({"string", "concatenated_string"})
-_PY_AUG_ASSIGN_KINDS: frozenset[str] = frozenset({"augmented_assignment"})
-
-# Heavy-resource constructors: building one of these per loop iteration opens a
-# fresh connection / client / pool instead of reusing a hoisted one. Keyed as
-# ``(root, method)`` pairs whose root is a distinctive I/O library (so the match
-# is unambiguous without import resolution) ...
-_PY_RESOURCE_CTORS: frozenset[tuple[str, str]] = frozenset(
- {
- ("sqlite3", "connect"),
- ("psycopg", "connect"),
- ("psycopg2", "connect"),
- ("pymysql", "connect"),
- ("MySQLdb", "connect"),
- ("aiosqlite", "connect"),
- ("httpx", "Client"),
- ("httpx", "AsyncClient"),
- ("requests", "Session"),
- ("aiohttp", "ClientSession"),
- ("boto3", "client"),
- ("boto3", "resource"),
- ("redis", "Redis"),
- ("redis", "StrictRedis"),
- ("pymongo", "MongoClient"),
- }
-)
-# ... plus a few constructors distinctive enough on their own (the rightmost
-# name), so an aliased ``from sqlalchemy import create_engine`` still resolves.
-_PY_RESOURCE_METHODS: frozenset[str] = frozenset({"create_engine", "MongoClient"})
+if TYPE_CHECKING:
+ from tree_sitter import Node
-# Lock acquisition (the contention side only — never ``release``, which would
-# double-count the same critical section). ``.acquire()`` is the threading /
-# asyncio / multiprocessing / filelock primitive verb.
-_PY_LOCK_METHODS: frozenset[str] = frozenset({"acquire"})
-# RHS node kinds that prove a name is bound to a list (the membership gate).
-_PY_LIST_RHS_KINDS: frozenset[str] = frozenset({"list", "list_comprehension"})
-# Builtins whose call result is provably a list.
-_PY_LIST_BUILTINS: frozenset[str] = frozenset({"list", "sorted"})
-# RHS shapes that make a name a NON-list container (a set/dict literal or
-# comprehension, or a ``set()`` / ``dict()`` / ``frozenset()`` call). A name
-# bound to one of these ANYWHERE in the file is ambiguous and excluded from the
-# membership gate, even if another scope binds the same name to a list — the
-# ``seen = []`` in one function / ``seen: set = set()`` in another collision.
-_PY_NONLIST_RHS_KINDS: frozenset[str] = frozenset(
- {"set", "dictionary", "set_comprehension", "dictionary_comprehension"}
-)
-_PY_NONLIST_BUILTINS: frozenset[str] = frozenset({"set", "dict", "frozenset"})
-
-# Promotion facts (perf/loop_facts.py). Calls and attributes that read a query
-# result out without changing how many rows it holds.
-_PY_RESULT_PROJECTIONS: frozenset[str] = frozenset(
- {"all", "scalars", "fetchall", "json", "data", "rows"}
-)
-# Raw SQL capped in its own text: ``sa.text("... LIMIT 1000")``.
-_PY_SQL_LIMIT_RE = re.compile(rb"(?is)\bselect\b.*\blimit\s+\d+\b")
-_PY_SQL_COMMENT_RE = re.compile(rb"--[^\n]*|/\*.*?\*/", re.S)
-# A per-key call that limits or orders its rows is not the read one IN query makes.
-_PY_PER_KEY_LIMITS: frozenset[str] = frozenset(
- {
- "limit", "single", "maybe_single", "first", "one", "scalar_one",
- "scalar_one_or_none", "order", "order_by", "range", "offset",
- }
-)
-_PY_ONE_ROW: frozenset[str] = _PY_PER_KEY_LIMITS | {"scalar"}
-_PY_READ_CAPS: frozenset[str] = _PY_ONE_ROW - {"order", "order_by", "offset"}
-_PY_WRITES: frozenset[str] = frozenset({"update", "insert", "upsert", "values"})
-# Keys a query yields (``.in_(select(...))``) are as many as that table's rows.
-_PY_KEY_READS: frozenset[str] = _PY_STATEMENT_BUILDERS | {
- "subquery", "scalar_subquery", "query", "cte", "alias", "union", "union_all",
- "execute", "table", "from_",
+_PY_DB_METHODS: frozenset[str] = PY_DB_UNAMBIGUOUS | PY_DB_NEEDS_EVIDENCE | PY_DB_AMBIGUOUS
+_PY_ROOTED_PROCESS_VERBS: dict[str, frozenset[str]] = {"subprocess": PY_SUBPROC_METHODS}
+_PY_ROOTED_FILE_VERBS: dict[str, frozenset[str]] = {
+ "open": frozenset({"open"}),
+ "os": PY_OS_FS_METHODS,
+ "shutil": PY_SHUTIL_FS_METHODS,
}
-_PY_SCOPES: frozenset[str] = frozenset({"function_definition", "lambda", "class_definition"})
-_PY_BUILTINS: frozenset[str] = frozenset(dir(builtins))
-_PY_SEM_NAME_RE = re.compile(r"(?i)(sem|semaphore|limiter|limit)$")
-# A model class is CapWords; an ALL_CAPS constant (``API_URL``) is not.
-_PY_MODEL_NAME_RE = re.compile(r"^[A-Z]\w*[a-z]\w*$")
-# Directory listings; the recursive ones walk a whole tree.
-_PY_LISTINGS: frozenset[str] = frozenset(
- {
- "glob", "iglob", "rglob", "iterdir", "walk", "listdir", "scandir", "traverse",
- "iter_glob", "glob_via", "walk_repo", "_rglob",
- }
-)
-_PY_RECURSIVE_LISTINGS: frozenset[str] = frozenset(
- {"rglob", "walk", "traverse", "walk_repo", "_rglob"}
-)
-# Reads that yield lines or chunks as they arrive.
-_PY_STREAMS: frozenset[str] = frozenset(
- {
- "readlines", "iter_lines", "iter_content", "iter_bytes", "iter_text",
- "aiter_lines", "aiter_bytes", "aiter_text", "aiter_raw",
- }
-)
-# Wrappers and chunkers whose first argument is what the loop walks.
-_PY_PASS_THROUGH: frozenset[str] = frozenset(
- {
- "enumerate", "sorted", "reversed", "list", "set", "tuple", "iter", "zip",
- "batched", "chunked", "ichunked", "chunks", "iter_chunks", "grouper",
- }
-)
-_PY_CONSTANT_RHS: frozenset[str] = frozenset({"list", "tuple", "set", "dictionary"})
-
-
-_PY_DEF_KINDS = frozenset({"function_definition", "lambda", "class_definition"})
-# Nodes an identifier sits in when it is the name being bound, not read.
-_PY_BINDING_PARENTS = frozenset(
- {
- "global_statement", "nonlocal_statement", "as_pattern_target", "aliased_import",
- "import_from_statement", "import_statement", "parameters", "lambda_parameters",
- "named_expression",
- }
-)
-_PY_BINDING_WRAPPERS = frozenset(
- {
- "pattern_list", "tuple_pattern", "list_pattern", "typed_parameter", "default_parameter",
- "typed_default_parameter", "list_splat_pattern", "dotted_name",
- }
-)
-
-
-def _binds(ident: Node) -> bool:
- """Whether *ident* is a name being bound (assigned, imported, a parameter)."""
- cur, parent = ident, ident.parent
- while parent is not None and parent.type in _PY_BINDING_WRAPPERS:
- cur, parent = parent, parent.parent
- if parent is None:
- return False
- if parent.type in ("assignment", "augmented_assignment", "for_statement", "for_in_clause"):
- return parent.child_by_field_name("left") == cur
- if parent.type in ("function_definition", "class_definition"):
- return parent.child_by_field_name("name") == cur
- return parent.type in _PY_BINDING_PARENTS
-
-
-def _bindings(scope: Node, name: bytes) -> list[Node]:
- """Identifiers binding *name* in *scope* itself, not in a nested scope."""
- found: list[Node] = []
- stack = list(scope.children)
- while stack:
- node = stack.pop()
- if node.type in _PY_DEF_KINDS:
- stack.extend(c for c in (node.child_by_field_name("name"),) if c is not None)
- continue
- if node.type == "identifier" and node.text == name and _binds(node):
- found.append(node)
- stack.extend(node.children)
- return found
-def _module_int_constant(name: Node) -> bool:
- """An ALL_CAPS name its own module binds exactly once, to an integer literal
- (``MAX = 5``), and no enclosing function rebinds or shadows.
-
- The spelling alone proves nothing (``NUM_USERS`` grows), so a name rebound,
- imported, augmented, written through ``global``, shadowed by a parameter or
- a local, or bound to anything but a literal stays unknown.
- """
- text = name.text or b""
- if name.type != "identifier" or len(text) < 2 or not text.decode().isupper():
- return False
- scope = name.parent
- while scope is not None and scope.parent is not None:
- if scope.type in ("function_definition", "lambda") and _bindings(scope, text):
- return False
- scope = scope.parent
- if scope is None or any(
- n.type == "global_statement" and any(c.text == text for c in n.named_children)
- for n in BasePerfDialect._walk(scope)
- ):
- return False
- bound = _bindings(scope, text)
- if len(bound) != 1 or bound[0].parent is None or bound[0].parent.type != "assignment":
- return False
- right = bound[0].parent.child_by_field_name("right")
- return right is not None and right.type == "integer"
-
-
-def _text(node: Node) -> str:
- return (node.text or b"").decode("utf-8", "replace")
-
-
-def _field_text(node: Node, field: str) -> str:
- child = node.child_by_field_name(field)
- return _text(child) if child is not None else ""
-
-
-def _value_refs(expr: Node) -> list[Node]:
- """The names and dotted paths *expr* reads values from: not a callee
- (``tuple`` in ``tuple(xs)``), a keyword, or an ALL_CAPS constant."""
- refs = []
- stack = [expr]
- while stack:
- node = stack.pop()
- if node.type not in ("identifier", "attribute"):
- stack.extend(node.children)
- continue
- parent = node.parent
- callee = parent is not None and parent.child_by_field_name("function") == node
- keyword = parent is not None and parent.type == "keyword_argument" and (
- parent.child_by_field_name("name") == node
- )
- last = node.child_by_field_name("attribute") if node.type == "attribute" else node
- constant = last is not None and _text(last).strip("_").isupper()
- if callee and node.type == "attribute":
- # ``self.ids.copy()`` still reads ``self.ids``.
- stack.extend(filter(None, [node.child_by_field_name("object")]))
- elif not (callee or keyword or constant):
- refs.append(node)
- return refs
-
-
-def _string_value(node: Node) -> str | None:
- return _text(node).strip("\"'") if node.type == "string" else None
-
-
-def _in_filter(call: Node, args: list[Node]) -> tuple[str, Node] | None:
- """``(column, keys)`` of ``Model.col.in_(keys)`` or PostgREST ``.in_("col", keys)``."""
- if len(args) == 2:
- column = _string_value(args[0])
- return (column, args[1]) if column else None
- fn = call.child_by_field_name("function")
- receiver = fn.child_by_field_name("object") if fn is not None else None
- if len(args) != 1 or receiver is None or receiver.type != "attribute":
+def _db_sink(method: str, is_attribute: bool, evidence: bool) -> str | None:
+ """A DB sink is a member call on a session or cursor (never the builtin
+ ``all(...)``); verbs other clients also spell need db evidence."""
+ if not is_attribute:
return None
- column = _field_text(receiver, "attribute")
- return (column, args[0]) if column else None
-
-
-def _fixed_set(value: Node | None) -> bool:
- """A non-empty literal of scalars; an empty one is a container filled later."""
- return (
- value is not None
- and value.type in _PY_CONSTANT_RHS
- and bool(value.named_children)
- and all(_scalar_literal(item) for item in value.named_children)
- )
-
-
-_PY_SCALARS = frozenset({"string", "concatenated_string", "integer", "float", "true", "false", "none"})
+ return "db" if method in PY_DB_UNAMBIGUOUS or evidence else None
-def _scalar_literal(item: Node) -> bool:
- """``"a"`` / ``3`` / ``("a", 1)`` / ``"k": v`` by its key: an entry the source
- spells out. A name or call in the literal (``[ROOT / "a", repo.path]``) may
- stand for anything, so the table is not shown to be a fixed set."""
- if item.type == "pair":
- key = item.child_by_field_name("key")
- return key is not None and key.type in _PY_SCALARS
- if item.type == "tuple":
- return all(child.type in _PY_SCALARS for child in item.named_children)
- return item.type in _PY_SCALARS
-
-
-class PythonPerfDialect(IterableSources):
+class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):
language = "python"
lock_acquire_functions = frozenset({"acquire", "__enter__"})
spin_loop_header = re.compile(r"while\s+True\s*:")
@@ -403,14 +125,8 @@ class PythonPerfDialect(IterableSources):
string_literal_kinds = _PY_STRING_KINDS
aug_assign_kinds = _PY_AUG_ASSIGN_KINDS
- # Everything whose cost is bounded by ONE inode: a single file read/write, a
- # single unlink/mkdir/rename, one directory listing. Outside a loop that is
- # ordinary work, not a latency risk, so it must not reach ``hot_path_sync_io``
- # (see ``BasePerfDialect.hot_path_excluded_methods``). What survives is the
- # unbounded set — a whole-tree walk, a recursive copy/delete, an archive
- # round-trip — plus the bare ``open(...)`` builtin the marker was originally
- # calibrated on. Keyed on the method name alone, so a same-named non-``os``
- # receiver is excluded too: recall-only, never a false positive.
+ # One-inode operations are ordinary work outside a loop; only unbounded ones
+ # (tree walks, recursive copy/delete, archives) and bare ``open`` stay hot-path I/O.
hot_path_excluded_methods = (
PY_FS_METHODS
| (PY_OS_FS_METHODS - {"walk"})
@@ -427,53 +143,22 @@ class PythonPerfDialect(IterableSources):
io_names: dict[str, str],
has_db_import: bool,
) -> str | None:
- root_kind = io_names.get(root)
- db_evidence = has_db_import or root_kind == "db"
-
- # ``asyncio.sleep`` / ``time.sleep`` / ``trio.sleep`` is a cooperative
- # yield, never an I/O round-trip — but ``await asyncio.sleep(...)`` would
- # otherwise hit the awaited-network arm below when ``asyncio`` is
- # import-classified as network, FP-ing ``io_in_loop`` / ``serial_await``
- # on every backoff/poll loop, measured across a Python corpus. No legitimate
- # execution sink is named ``sleep``, so excluding it costs no recall.
+ # ``sleep`` is a cooperative yield, never a round-trip, even when its
+ # module is import-classified as network.
if method == "sleep":
return None
-
- if method in PY_DB_UNAMBIGUOUS:
- # A real DB sink is always a method on a session/cursor/result
- # object; a bare identifier call (the builtin ``all(...)``) is not.
- return "db" if is_attribute else None
- if method in PY_DB_NEEDS_EVIDENCE:
- # A DB verb only with db evidence in the file / on the receiver.
- return "db" if (is_attribute and db_evidence) else None
- if method in PY_DB_AMBIGUOUS:
- # ``.all`` / ``.first`` / ``.one`` collide with ordinary collection
- # helpers — the riskiest stratum. Gate on db evidence.
- return "db" if (is_attribute and db_evidence) else None
- if root == "subprocess" and method in PY_SUBPROC_METHODS:
- return "subprocess"
- if method == "Popen":
+ root_kind = io_names.get(root)
+ if method in _PY_DB_METHODS:
+ return _db_sink(method, is_attribute, has_db_import or root_kind == "db")
+ if method == "Popen" or method in _PY_ROOTED_PROCESS_VERBS.get(root, ()):
return "subprocess"
- if root == "open" and method == "open": # bare ``open(...)`` builtin
- return "filesystem"
- if is_attribute and method in PY_FS_METHODS:
- # ``p.read_text()`` / ``Path(x).write_bytes(...)``. ``is_attribute``
- # mirrors the DB branches: pathlib always spells these as a member
- # call, so requiring it costs no recall and drops a bare local
- # helper that happens to share the name.
- return "filesystem"
- if root == "os" and method in PY_OS_FS_METHODS:
- return "filesystem"
- if root == "shutil" and method in PY_SHUTIL_FS_METHODS:
+ # A verb under its literal module root (bare ``open`` too), or a pathlib member call.
+ if method in _PY_ROOTED_FILE_VERBS.get(root, ()) or (is_attribute and method in PY_FS_METHODS):
return "filesystem"
- if method == "urlopen":
- return "network"
- if is_attribute and root_kind == "network" and method in HTTP_VERBS:
- # ``requests.get(...)`` is a member call on an imported client; a bare
- # ``get(...)`` (e.g. a builtin/local) never has the verb-call shape, so
- # gate on ``is_attribute`` like the DB branches above.
- return "network"
- if awaited and root_kind == "network":
+ # An HTTP verb only as a member call on an imported client: a bare ``get(...)`` is not one.
+ if method == "urlopen" or (
+ root_kind == "network" and (awaited or (is_attribute and method in HTTP_VERBS))
+ ):
return "network"
return None
@@ -561,16 +246,22 @@ class PythonPerfDialect(IterableSources):
return False
loop = loop.parent
body = loop.child_by_field_name("body") if loop is not None else None
+ if body is None:
+ return False
session = (self.callee_root_name(call) or "").encode()
- for node in self._walk(body, prune=_PY_SCOPES) if body is not None else ():
- args = node.child_by_field_name("arguments") if node.type == "call" else None
- if args is None or node.end_byte > call.start_byte:
- continue
- texts = {arg.text for arg in args.children}
- same_session = session in texts or self.callee_root_name(node) == session.decode()
- if key.text in texts and same_session:
- return True
- return False
+ return any(
+ self._passes_key(node, call, key, session)
+ for node in self._walk(body, prune=_PY_SCOPES)
+ )
+
+ def _passes_key(self, node: Node, call: Node, key: Node, session: bytes) -> bool:
+ """*node* is a call before *call* passing *key* on the same *session*."""
+ args = node.child_by_field_name("arguments") if node.type == "call" else None
+ if args is None or node.end_byte > call.start_byte:
+ return False
+ texts = {arg.text for arg in args.children}
+ same_session = session in texts or self.callee_root_name(node) == session.decode()
+ return key.text in texts and same_session
def _orm_get(self, call: Node) -> tuple[str, Node] | None:
"""``(Model, key)`` of a ``x.get(Model, key)`` primary-key lookup."""
@@ -587,12 +278,8 @@ class PythonPerfDialect(IterableSources):
return model, positional[1]
def is_constant_loop(self, node: Node) -> bool:
- """True if a Python for-loop iterates a compile-time-constant bound.
-
- Catches ``for _ in range(<int literals>)``, ``for x in (<literal>)``,
- and ``for x in ALL_CAPS`` (a named module constant by convention).
- ``while`` loops are never constant.
- """
+ """``for _ in range(<int literals>)``, ``for x in (<literal>)`` or
+ ``for x in ALL_CAPS`` (a module constant by convention)."""
if node.type != "for_statement":
return False
right = node.child_by_field_name("right")
@@ -600,28 +287,11 @@ class PythonPerfDialect(IterableSources):
return False
if right.type in ("list", "tuple", "set"):
return True
- # A bare ALL_CAPS identifier is a named constant by convention.
if right.type == "identifier" and right.text is not None:
name = right.text.decode("utf-8", "replace")
if name.isupper() and len(name) > 1:
return True
- if right.type == "call":
- fn = right.child_by_field_name("function")
- if fn is None or (fn.text or b"").decode("utf-8", "replace") != "range":
- return False
- args = right.child_by_field_name("arguments")
- if args is None:
- return False
- for a in args.children:
- if not a.is_named:
- continue
- if a.type == "integer":
- continue
- if a.type == "unary_operator" and any(c.type == "integer" for c in a.children):
- continue
- return False # a non-literal arg (e.g. len(x)) ⇒ data-dependent
- return True
- return False
+ return right.type == "call" and _literal_range(right)
# Builtins that wrap a collection without changing what is iterated, so the
# same-collection O(n^2) shape sees through them: ``for x in enumerate(items)``
@@ -693,16 +363,8 @@ class PythonPerfDialect(IterableSources):
return len(named) == 3 and named[2].text not in (b"1", b"-1")
def is_string_concat(self, node: Node) -> bool:
- """``s += "x"`` accumulation — but skip an accumulator that is *reset*
- each iteration of an enclosing loop.
-
- ``buf = base[:N]; ... buf += part`` inside a loop builds a fresh, bounded
- string per iteration (not the O(n^2) cross-iteration accumulation the
- marker targets), so it is a false positive. Measured on a Python corpus:
- the reset-per-iteration shape was the dominant FP class (Py 77.8%). When
- the same name is plainly re-assigned inside an enclosing loop body, the
- ``+=`` cannot accumulate across iterations, so do not flag it.
- """
+ """``s += "x"`` accumulation, unless an enclosing loop body re-assigns the
+ name: then the string is rebuilt per iteration, not accumulated across them."""
if not super().is_string_concat(node):
return False
left = node.child_by_field_name("left")
@@ -766,21 +428,11 @@ class PythonPerfDialect(IterableSources):
) -> str | None:
if (root, method) in _PY_RESOURCE_CTORS or method in _PY_RESOURCE_METHODS:
return "resource_construction_in_loop"
- # ``lock.acquire()`` — a method call on a receiver (never the builtin
- # ``acquire(...)``, which does not exist; the attribute gate is a cheap
- # extra guard against a bare-name collision).
+ # ``lock.acquire()``: the attribute gate guards against a bare-name collision.
if method in _PY_LOCK_METHODS and self.callee_is_attribute(node):
return "lock_in_loop"
- # ``lst.insert(0, x)`` each iteration shifts the whole list -> O(n^2);
- # ``collections.deque.appendleft`` / build-then-reverse is O(n). Gated to
- # a literal ``0`` first arg (``insert(i, x)`` at a variable index is not
- # the front-insertion anti-pattern) AND to a list that is not re-created
- # each iteration (a fresh ``buf = [...]; buf.insert(0, x)`` is bounded,
- # not O(n^2) — the same reset-per-iteration FP class as string_concat;
- # measured on a Python corpus).
- # ``collections.deque.insert(0, x)`` is O(1) front-insertion by design (it
- # is the deque's own ``appendleft`` machinery), so exclude a receiver
- # provably constructed via ``deque()`` / ``collections.deque(...)``.
+ # ``lst.insert(0, x)`` shifts the whole list each pass: a literal ``0``, on a
+ # list not re-created per iteration and not a ``deque`` (O(1) at the front).
if (
method == "insert"
and self.callee_is_attribute(node)
@@ -796,18 +448,10 @@ class PythonPerfDialect(IterableSources):
return None
def loop_iterable_call_marker(self, node: Node) -> str | None:
- """``for _, row in df.iterrows():`` — row-by-row DataFrame iteration.
+ """``for _, row in df.iterrows():``, row-by-row DataFrame iteration in the loop header.
- ``DataFrame.iterrows()`` boxes every row into a fresh Series, an order of
- magnitude slower than a vectorized operation (and slower than
- ``itertuples`` / ``to_dict`` when row access is unavoidable). The call
- sits in the loop header, so the body ``loop_call_marker`` misses it.
- Gated to the distinctive method name on a member-access receiver
- (``x.iterrows()``, never a bare ``iterrows(...)``) AND to a file that
- imports pandas (same style as the ``pd``/``pandas`` root gate in
- ``pd_concat``). Residual limitation: this is still a name match — a
- user-defined class with an ``iterrows`` method in a file that also
- imports pandas would still be flagged.
+ A member call in a file that imports pandas. Ceiling: still a name match,
+ so a user class with an ``iterrows`` method in such a file is flagged too.
"""
if node.type != "for_statement":
return None
@@ -927,16 +571,9 @@ class PythonPerfDialect(IterableSources):
return "membership_test_against_list_in_loop" if name in list_names else None
def list_bound_names(self, root: Node) -> frozenset[str]:
- """Names assigned a provable list anywhere in the file, minus any name
- also bound to a non-list container.
-
- Covers ``name = [...]`` / ``name = [x for x in ...]`` / ``name =
- list(...)`` / ``name = sorted(...)``. Conservative on purpose: an opaque
- ``name = build()`` is not counted, and a name bound to a set/dict in any
- scope of the file is dropped (the ``seen``-as-list-here /
- ``seen``-as-set-there collision), so the membership marker only fires
- against a name we can prove is always a list.
- """
+ """Names assigned a provable list anywhere in the file (``[...]``, a list
+ comprehension, ``list(...)``, ``sorted(...)``), minus any name also bound to
+ a set or dict, or used as a parameter, in any scope."""
list_names: set[str] = set()
exclude: set[str] = set()
stack: list[Node] = [root]
@@ -957,10 +594,6 @@ class PythonPerfDialect(IterableSources):
elif self._rhs_is_nonlist_container(right):
exclude.add(name)
elif n.type == "parameters":
- # A name also used as a function parameter has no proven binding
- # inside that scope (the caller could pass a set), so it collides
- # with an unrelated module-level list of the same name. Exclude it,
- # mirroring the set/dict-literal exclusion above.
exclude.update(self._param_names(n))
for c in n.children:
stack.append(c)
@@ -1054,438 +687,5 @@ class PythonPerfDialect(IterableSources):
ok = target is not None and target.type == "identifier" and target.text and value is not None
return [(target.text, value)] if ok else []
- def loop_source(self, loop: Node, probe: SinkProbe) -> LoopSource | None:
- if loop.type != "for_statement":
- return None
- right = loop.child_by_field_name("right")
- baseline = self._magnitude(right, loop, probe, hop=True) or "unknown"
- iterates, claim = self._source(right, loop, probe, hop=True)
- if iterates == "unknown" and baseline == "unknown":
- return None
- return LoopSource(iterates, claim or baseline)
-
- def _source(self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
- """What *expr* walks, and the magnitude that settles when it differs from
- what the iterable's syntax alone proves (``None``: keep that).
-
- *hop* allows one step through the name's last assignment before the loop.
- """
- if expr is None:
- return "unknown", None
- if expr.type in ("await", "parenthesized_expression"):
- inner = next((c for c in expr.children if c.is_named), None)
- return self._source(inner, loop, probe, hop=hop)
- if expr.type == "boolean_operator" and any(c.type == "or" for c in expr.children):
- return self._source(expr.child_by_field_name("left"), loop, probe, hop=hop)
- if expr.type in ("identifier", "attribute") and lex.is_config_name(_text(expr)):
- # A registry grows with what users register: the name alone bounds nothing.
- return "config_table", None
- if expr.type == "identifier":
- return self._name_source(expr, loop, probe, hop=hop)
- if expr.type == "attribute":
- return self._attribute_source(expr, loop, probe, hop=hop)
- if expr.type == "subscript" and self._slice_magnitude(expr) == "bounded":
- # A fixed-width slice bounds the trip count, not what is walked.
- walked = self._source(expr.child_by_field_name("value"), loop, probe, hop=hop)
- return walked[0], "bounded"
- if expr.type == "call":
- return self._call_source(expr, loop, probe, hop=hop)
- return "unknown", None
-
- def _attribute_source(self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
- """``result.rows`` walks ``result``; ``self._TABLES`` bound to a literal is a constant."""
- attr = expr.child_by_field_name("attribute")
- name = (attr.text if attr is not None else b"") or b""
- if name.decode("utf-8", "replace") in _PY_RESULT_PROJECTIONS:
- return self._source(expr.child_by_field_name("object"), loop, probe, hop=hop)
- if self._literal_constant(expr, name):
- return "constant", "bounded"
- return "unknown", None
-
- def _name_source(self, name: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
- rhs = self.reaching_value(loop, name.text or b"") if hop else None
- if rhs is not None:
- return self._source(rhs, loop, probe, hop=False)
- fn = loop.parent
- while fn is not None and fn.type != "function_definition":
- fn = fn.parent
- params = fn.child_by_field_name("parameters") if fn is not None else None
- if params is not None and _text(name) in self._param_names(params):
- return "parameter", None
- return "unknown", None
-
- def _call_source(self, call: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
- method, fn, receiver, args = self._call_bits(call)
- if lex.is_config_name(self._dotted_path(fn) or "", callee=True):
- return "config_table", None
- if method == "range" and receiver is None:
- return self._range_source(call, args, loop, probe, hop=hop)
- walked = self._walked_arg_or_receiver(method, receiver, args)
- if walked is not None:
- return self._source(walked, loop, probe, hop=hop)
- if method in _PY_LISTINGS or method in _PY_STREAMS or (method == "open" and not receiver):
- return self._file_source(call, method, loop)
- if probe(call) in ("db", "network"):
- return self._query(call, loop)
- if receiver is not None and method in _PY_RESULT_PROJECTIONS:
- return self._source(receiver, loop, probe, hop=hop)
- return "unknown", None
-
- def _call_bits(self, call: Node) -> tuple[str, Node | None, Node | None, list[Node]]:
- """``(method, callee, receiver, positional args)`` of a call."""
- fn = call.child_by_field_name("function")
- receiver = fn.child_by_field_name("object") if fn is not None and fn.type == "attribute" else None
- arg_list = call.child_by_field_name("arguments")
- args = [c for c in arg_list.children if c.is_named] if arg_list is not None else []
- return self.callee_method_name(call) or "", fn, receiver, args
-
- @staticmethod
- def _walked_arg_or_receiver(
- method: str, receiver: Node | None, args: list[Node]
- ) -> Node | None:
- """``sorted(xs)`` / ``enumerate(xs)`` walk ``xs``; ``d.items()`` walks ``d``."""
- if method in _PY_PASS_THROUGH and args and args[0].type != "keyword_argument":
- return args[0]
- if receiver is not None and method in ("items", "values", "keys"):
- return receiver
- return None
-
- def _file_source(self, call: Node, method: str, loop: Node) -> Source:
- """A directory listing (grows when it walks the repository tree) or a stream."""
- if method in _PY_LISTINGS:
- return "listing", "grows_with_data" if self._walks_repo(call, method, loop) else None
- return "stream", None
-
- def _range_source(
- self, call: Node, args: list[Node], loop: Node, probe: SinkProbe, *, hop: bool
- ) -> Source:
- """``range(3)`` is a constant; ``range(0, len(ids), N)`` walks ``ids``."""
- if self._range_magnitude(call, loop, probe, hop=hop) == "bounded":
- return "constant", "bounded"
- stop = args[1] if len(args) > 1 else (args[0] if args else None)
- if stop is not None and stop.type == "call" and self.callee_method_name(stop) == "len":
- inner = [c for c in stop.child_by_field_name("arguments").children if c.is_named]
- if len(inner) == 1:
- return self._source(inner[0], loop, probe, hop=hop)
- return "unknown", None
-
- def reads_config_table(self, call: Node, nodes: list[Node]) -> bool:
- """Every SQL text and selected model names a registry or configuration table."""
- walked = [n for part in nodes for n in self._walk(part)]
- sql = [
- n.text or b""
- for n in walked
- if n.type == "string" and _PY_SQL_LITERAL_RE.match(_text(n))
- ]
- models = [
- model
- for n in walked
- if n.type == "call" and self.callee_method_name(n) in ("select", "query")
- for model in self._selected_models(n)
- ]
- return bool(sql or models) and (
- all(lex.is_config_sql(text) for text in sql)
- and all(lex.is_config_model(model) for model in models)
- )
-
- def _selected_models(self, select: Node) -> list[str]:
- """``select(Repository)`` / ``query(Repository.id, Repository.path)`` -> the model
- names; ``func.count()`` and other expressions name none."""
- args = select.child_by_field_name("arguments")
- models: list[str] = []
- for arg in (c for c in args.children if c.is_named) if args is not None else ():
- path = self._dotted_path(arg) or ""
- head = next((seg for seg in path.split(".") if _PY_MODEL_NAME_RE.match(seg)), None)
- if head is not None:
- models.append(head)
- return models
-
- def _literal_constant(self, expr: Node, name: bytes) -> bool:
- """``self._TABLES`` where the file binds ``_TABLES`` once, to a literal collection."""
- text = name.decode("utf-8", "replace")
- if len(text.strip("_")) < 2 or not text.isupper():
- return False
- root = expr
- while root.parent is not None:
- root = root.parent
- rights = [
- n.child_by_field_name("right")
- for n in self._walk(root)
- if n.type == "assignment"
- and (left := n.child_by_field_name("left")) is not None
- and left.type == "identifier"
- and left.text == name
- ]
- return (
- len(rights) == 1
- and rights[0] is not None
- and rights[0].type in _PY_CONSTANT_RHS
- and all(_scalar_literal(item) for item in rights[0].named_children)
- )
-
- def _magnitude(
- self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool
- ) -> LoopMagnitude | None:
- """Grows (a query result or a directory listing), bounded, or ``None``.
-
- *hop* allows one step through the name's last assignment before the
- loop; a parameter has none and stays unknown on purpose.
- """
- if expr is None:
- return None
- if expr.type in ("await", "parenthesized_expression"):
- inner = next((c for c in expr.children if c.is_named), None)
- return self._magnitude(inner, loop, probe, hop=hop)
- if expr.type == "identifier" and hop and expr.text:
- rhs = self.reaching_value(loop, expr.text)
- return self._magnitude(rhs, loop, probe, hop=False)
- if expr.type == "subscript":
- return self._slice_magnitude(expr)
- if expr.type == "boolean_operator" and any(c.type == "or" for c in expr.children):
- # ``(await q.execute()).data or []``: the fallback is empty, the read is not.
- left = expr.child_by_field_name("left")
- return self._grows(self._magnitude(left, loop, probe, hop=hop))
- if expr.type == "attribute":
- attr = expr.child_by_field_name("attribute")
- if attr is None or (attr.text or b"").decode() not in _PY_RESULT_PROJECTIONS:
- return None
- return self._grows(self._magnitude(expr.child_by_field_name("object"), loop, probe, hop=hop))
- return self._call_magnitude(expr, loop, probe, hop=hop) if expr.type == "call" else None
-
- def _call_magnitude(
- self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool
- ) -> LoopMagnitude | None:
- method = self.callee_method_name(expr) or ""
- member = self.callee_is_attribute(expr)
- if method == "range" and not member:
- return self._range_magnitude(expr, loop, probe, hop=hop)
- if probe(expr) in ("db", "network"):
- # A read capped in the query is as large as its cap; a statement or result
- # built before the loop (``stmt = ....limit(n)``) is read one assignment back.
- names = [n.text for n in self._walk(expr) if n.type == "identifier"]
- parts = [expr, *filter(None, (self.reaching_value(loop, name) for name in names))]
- capped = any(self.is_cap(n, loop) for part in parts for n in self._walk(part))
- return None if capped else "grows_with_data"
- if member and method in _PY_RESULT_PROJECTIONS:
- receiver = expr.child_by_field_name("function").child_by_field_name("object")
- return self._grows(self._magnitude(receiver, loop, probe, hop=hop))
- return None
-
- def is_cap(self, node: Node, loop: Node) -> bool:
- """``.limit(n)`` and peers, a SQL string saying ``LIMIT n``, or a filter to a
- set of keys (:meth:`read_key_column`), which is as large as the keys are."""
- if node.type == "string":
- return bool(_PY_SQL_LIMIT_RE.search(_PY_SQL_COMMENT_RE.sub(b"", node.text or b"")))
- method = self.callee_method_name(node) if node.type == "call" else None
- if method in _PY_READ_CAPS:
- return True
- return node.type == "call" and self.read_key_column(node, loop) is not None
-
- def read_key_column(self, call: Node, anchor: Node) -> str | None:
- """``Model.col.in_(keys)`` and PostgREST ``.in_("col", keys)`` -> ``col``;
- ``filter_by(id=k)`` and ``.eq("id", k)`` -> ``id``. Keys a query yields
- (``.in_(select(...))``, or a name bound to one) grow with that table."""
- method = self.callee_method_name(call)
- arg_list = call.child_by_field_name("arguments")
- args = arg_list.named_children if arg_list is not None else []
- if method == "filter_by":
- named = {_field_text(a, "name") for a in args if a.type == "keyword_argument"}
- return "id" if "id" in named else None
- if method == "eq":
- return "id" if len(args) == 2 and _string_value(args[0]) == "id" else None
- found = _in_filter(call, args) if method == "in_" else None
- if found is None or not self._handed_in(found[1], anchor):
- return None
- return found[0]
-
- def _handed_in(self, keys: Node, anchor: Node) -> bool:
- """*keys* holds values given to the code, not ones a query yields or a fixed
- set of states (``in_(("a", "b"))``, ``in_(SOURCES)``) whose rows grow."""
- values = [
- self.reaching_value(anchor, ref.text or b"") if ref.type == "identifier" else None
- for ref in _value_refs(keys)
- ]
- if all(_fixed_set(value) for value in values):
- return False
- parts = [keys, *filter(None, values)]
- return not any(
- n.type == "call" and self.callee_method_name(n) in _PY_KEY_READS
- for part in parts
- for n in self._walk(part)
- )
-
- def _range_magnitude(
- self, call: Node, loop: Node, probe: SinkProbe, *, hop: bool
- ) -> LoopMagnitude | None:
- """``range(len(rows))`` grows with ``rows``; ``range(3)`` and ``range(MAX)``
- with ``MAX = 3`` bound once in the module are bounded."""
- args = [c for c in call.child_by_field_name("arguments").children if c.is_named]
- if not args:
- return None
- stop = args[1] if len(args) > 1 else args[0]
- if stop.type == "call" and self.callee_method_name(stop) == "len":
- inner = [c for c in stop.child_by_field_name("arguments").children if c.is_named]
- if len(inner) == 1:
- return self._grows(self._magnitude(inner[0], loop, probe, hop=hop))
- return None
- # A name alone (``MAX_RETRIES``, ``self.max_attempts``) proves nothing.
- return "bounded" if stop.type == "integer" or _module_int_constant(stop) else None
-
- def _slice_magnitude(self, subscript: Node) -> LoopMagnitude | None:
- """``xs[:5]`` / ``xs[:LIMIT]`` / ``xs[i:i + 4]``: a constant-width slice."""
- piece = subscript.child_by_field_name("subscript")
- if piece is None or piece.type != "slice" or sum(c.type == ":" for c in piece.children) != 1:
- return None
- colon = next(i for i, c in enumerate(piece.children) if c.type == ":")
- start = next((c for c in piece.children[:colon] if c.is_named), None)
- stop = next((c for c in piece.children[colon + 1 :] if c.is_named), None)
- if stop is None:
- return None
- if start is None:
- return "bounded" if stop.type == "integer" or _module_int_constant(stop) else None
- if stop.type == "binary_operator":
- left, right = stop.child_by_field_name("left"), stop.child_by_field_name("right")
- same_start = left is not None and self._dotted_path(left) == self._dotted_path(start)
- if same_start and right is not None and right.type == "integer":
- return "bounded"
- return None
-
- def batch_form(self, sink: Node, loop: Node, probe: SinkProbe) -> BatchForm | None:
- target = loop.child_by_field_name("left")
- if loop.type != "for_statement" or target is None or target.type != "identifier":
- return None
- body = self.loop_body(loop) or loop
- iterable = self.loop_iterable_name(loop)
- for node in self._walk(body):
- if (
- node.type == "call"
- and self.callee_is_attribute(node)
- and self.callee_root_name(node) == iterable
- and self.callee_method_name(node) in self.sequence_appends
- ):
- return None # a worklist: its keys are not known before the loop
- refs = [n for n in self._walk(sink) if n.type == "identifier" and n.text == target.text]
- methods = {self.callee_method_name(n) for n in self._walk(sink) if n.type == "call"}
- if len(refs) != 1 or methods & _PY_WRITES or self._reads_loop_local(sink, body):
- return None
- call = self._keyed_filter(sink, refs[0], methods)
- if call is None:
- return None
- if "delete" in methods:
- # Batched, a delete also removes rows a skipped iteration would have left.
- kept = self._unconditional(sink, body)
- else:
- kept = not self._builds_in_order(body)
- if self._orm_get(sink) is not None:
- # ``get`` answers from the session's identity map when the row is already
- # loaded, which no syntax shows: held out, half of these sites made no
- # round trip. The bulk form is named, never proven.
- kept = False
- equivalent = (
- kept
- and not methods & _PY_PER_KEY_LIMITS
- and not self._limited_downstream(sink, body)
- and self._only_io_in_body(sink, body, probe)
- and not self._calls_for_effect(sink, body)
- )
- return BatchForm(call, equivalent)
-
- def _calls_for_effect(self, sink: Node, body: Node) -> bool:
- """Another statement calls something this function did not build as a list, set
- or dict (``run_task(doc)``, ``session.add(r)``, ``t = task.delay(d)``), which may
- write what the batched read would read before it ran."""
- for node in self._walk(body, prune=_PY_SCOPES):
- if node.type == "expression_statement":
- call, assigned = node.named_children[0], False
- elif node.type == "assignment":
- call, assigned = node.child_by_field_name("right"), True
- else:
- continue
- if call is not None and call.type == "await":
- call = next((c for c in call.children if c.is_named), None)
- if call is None or call.type != "call" or self._within(call, sink):
- continue
- if self._has_effect(call, assigned=assigned):
- return True
- return False
-
- def _has_effect(self, call: Node, *, assigned: bool) -> bool:
- """Not ``seen.add(x)`` on a local collection, and, when its result is kept, not a
- builtin (``str(r.id)``) or a method on a local (``res.first()``)."""
- if not self.callee_is_attribute(call):
- return not (assigned and self.callee_method_name(call) in _PY_BUILTINS)
- before = self.bindings_before(call, (self.callee_root_name(call) or "").encode())
- if not before:
- return True
- rhs = before[-1][1]
- return not (assigned or self._rhs_is_list(rhs) or self._rhs_is_nonlist_container(rhs))
-
- def _limited_downstream(self, sink: Node, body: Node) -> bool:
- """The result is cut to one row after the call (``(await q).first()``,
- ``res = await q`` then ``res.scalar_one()``), which one IN query does not do."""
- names: set[bytes | None] = set()
- cur = sink
- while cur.parent is not None and cur.parent != body:
- cur = cur.parent
- if cur.type == "call" and self.callee_method_name(cur) in _PY_ONE_ROW:
- return True
- left = cur.child_by_field_name("left") if cur.type == "assignment" else None
- if left is not None and left.type == "identifier":
- names.add(left.text)
- return any(
- n.type == "call"
- and self.callee_method_name(n) in _PY_ONE_ROW
- and (self.callee_root_name(n) or "").encode() in names
- for n in self._walk(body)
- )
-
- def _keyed_filter(self, sink: Node, ref: Node, methods: set[str | None]) -> str | None:
- """The bulk filter when *sink* selects or deletes by equality on *ref*."""
- orm_get = self._orm_get(sink)
- if orm_get is not None and self._is_key_of(orm_get[1], ref):
- return f"select({orm_get[0]}).where(inspect({orm_get[0]}).primary_key[0].in_(keys))"
- for node in self._walk(sink):
- if node.type == "call" and self.callee_method_name(node) == "eq":
- # supabase-py: ``.table(t).select(...).eq("col", key)``
- args = [c for c in node.child_by_field_name("arguments").children if c.is_named]
- if (
- len(args) == 2
- and args[0].type == "string"
- and self._is_key_of(args[1], ref)
- and ("select" in methods) != ("delete" in methods)
- ):
- column = next(
- (c.text.decode() for c in args[0].children if c.type == "string_content"),
- None,
- )
- return f'.in_("{column}", keys)' if column else None
- if node.type == "comparison_operator" and any(c.type == "==" for c in node.children):
- # SQLAlchemy: ``.where(M.col == key)`` / ``.filter(M.col == key)``
- sides = [c for c in node.children if c.is_named]
- if len(sides) != 2:
- continue
- for column, key in (sides, sides[::-1]):
- path = self._dotted_path(column)
- if path and "." in path and self._is_key_of(key, ref):
- return f"{path}.in_(keys)"
- return None
-
- def concurrency_bound(self, sink: Node, loop: Node) -> str | None:
- if self._exits_early(self.loop_body(loop) or loop):
- return None
- cur = sink.parent
- while cur is not None and cur != loop:
- if cur.type == "with_statement" and any(c.type == "async" for c in cur.children):
- clause = next((c for c in cur.children if c.type == "with_clause"), None)
- for item in clause.children if clause is not None else ():
- value = item.child_by_field_name("value") if item.type == "with_item" else None
- # A name, not a constructor: ``async with Semaphore(5)`` in the
- # loop makes a fresh bound per iteration, which bounds nothing.
- path = self._dotted_path(value) if value is not None else None
- if path and _PY_SEM_NAME_RE.search(path.rsplit(".", 1)[-1]):
- return path
- cur = cur.parent
- return None
-
DIALECT = PythonPerfDialect()
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_batch.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_batch.py
new file mode 100644
index 000000000..05e0c0bbe
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_batch.py
@@ -0,0 +1,188 @@
+"""Whether a per-item Python query in a loop has a single batched form."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from ..loop_facts import BatchForm
+from .python_lexicon import (
+ _PY_BUILTINS,
+ _PY_ONE_ROW,
+ _PY_PER_KEY_LIMITS,
+ _PY_SCOPES,
+ _PY_SEM_NAME_RE,
+ _PY_WRITES,
+)
+from .python_nodes import _string_content
+
+if TYPE_CHECKING:
+ from tree_sitter import Node
+
+ from ..loop_facts import SinkProbe
+
+
+class PythonBatchForms:
+ """Batched forms and concurrency bounds. Mixed into :class:`~.python.PythonPerfDialect`."""
+
+ def batch_form(self, sink: Node, loop: Node, probe: SinkProbe) -> BatchForm | None:
+ target = loop.child_by_field_name("left")
+ if loop.type != "for_statement" or target is None or target.type != "identifier":
+ return None
+ body = self.loop_body(loop) or loop
+ if self._appends_to_iterable(body, self.loop_iterable_name(loop)):
+ return None # a worklist: its keys are not known before the loop
+ methods = {self.callee_method_name(n) for n in self._walk(sink) if n.type == "call"}
+ call = self._batch_call(sink, body, target, methods)
+ if call is None:
+ return None
+ return BatchForm(call, self._batch_equivalent(sink, body, probe, methods))
+
+ def _appends_to_iterable(self, body: Node, iterable: str | None) -> bool:
+ return any(
+ node.type == "call"
+ and self.callee_is_attribute(node)
+ and self.callee_root_name(node) == iterable
+ and self.callee_method_name(node) in self.sequence_appends
+ for node in self._walk(body)
+ )
+
+ def _batch_call(
+ self, sink: Node, body: Node, target: Node, methods: set[str | None]
+ ) -> str | None:
+ """The bulk filter for a read keyed once on the loop target and nothing loop-local."""
+ refs = [n for n in self._walk(sink) if n.type == "identifier" and n.text == target.text]
+ if len(refs) != 1 or methods & _PY_WRITES or self._reads_loop_local(sink, body):
+ return None
+ return self._keyed_filter(sink, refs[0], methods)
+
+ def _batch_equivalent(
+ self, sink: Node, body: Node, probe: SinkProbe, methods: set[str | None]
+ ) -> bool:
+ if "delete" in methods:
+ # Batched, a delete also removes rows a skipped iteration would have left.
+ kept = self._unconditional(sink, body)
+ else:
+ kept = not self._builds_in_order(body)
+ if self._orm_get(sink) is not None:
+ # ``get`` may answer from the session's identity map, which no syntax
+ # shows, so the bulk form is named, never proven.
+ kept = False
+ return (
+ kept
+ and not methods & _PY_PER_KEY_LIMITS
+ and not self._limited_downstream(sink, body)
+ and self._only_io_in_body(sink, body, probe)
+ and not self._calls_for_effect(sink, body)
+ )
+
+ def _calls_for_effect(self, sink: Node, body: Node) -> bool:
+ """Another statement calls something this function did not build as a list, set
+ or dict (``run_task(doc)``, ``session.add(r)``, ``t = task.delay(d)``), which may
+ write what the batched read would read before it ran."""
+ for node in self._walk(body, prune=_PY_SCOPES):
+ if node.type == "expression_statement":
+ call, assigned = node.named_children[0], False
+ elif node.type == "assignment":
+ call, assigned = node.child_by_field_name("right"), True
+ else:
+ continue
+ if call is not None and call.type == "await":
+ call = next((c for c in call.children if c.is_named), None)
+ if call is None or call.type != "call" or self._within(call, sink):
+ continue
+ if self._has_effect(call, assigned=assigned):
+ return True
+ return False
+
+ def _has_effect(self, call: Node, *, assigned: bool) -> bool:
+ """Not ``seen.add(x)`` on a local collection, and, when its result is kept, not a
+ builtin (``str(r.id)``) or a method on a local (``res.first()``)."""
+ if not self.callee_is_attribute(call):
+ return not (assigned and self.callee_method_name(call) in _PY_BUILTINS)
+ before = self.bindings_before(call, (self.callee_root_name(call) or "").encode())
+ if not before:
+ return True
+ rhs = before[-1][1]
+ return not (assigned or self._rhs_is_list(rhs) or self._rhs_is_nonlist_container(rhs))
+
+ def _limited_downstream(self, sink: Node, body: Node) -> bool:
+ """The result is cut to one row after the call (``(await q).first()``,
+ ``res = await q`` then ``res.scalar_one()``), which one IN query does not do."""
+ names: set[bytes | None] = set()
+ cur = sink
+ while cur.parent is not None and cur.parent != body:
+ cur = cur.parent
+ if cur.type == "call" and self.callee_method_name(cur) in _PY_ONE_ROW:
+ return True
+ left = cur.child_by_field_name("left") if cur.type == "assignment" else None
+ if left is not None and left.type == "identifier":
+ names.add(left.text)
+ return any(
+ n.type == "call"
+ and self.callee_method_name(n) in _PY_ONE_ROW
+ and (self.callee_root_name(n) or "").encode() in names
+ for n in self._walk(body)
+ )
+
+ def _keyed_filter(self, sink: Node, ref: Node, methods: set[str | None]) -> str | None:
+ """The bulk filter when *sink* selects or deletes by equality on *ref*."""
+ orm_get = self._orm_get(sink)
+ if orm_get is not None and self._is_key_of(orm_get[1], ref):
+ return f"select({orm_get[0]}).where(inspect({orm_get[0]}).primary_key[0].in_(keys))"
+ for node in self._walk(sink):
+ if node.type == "call" and self.callee_method_name(node) == "eq":
+ column_arg = self._eq_column_arg(node, ref, methods)
+ if column_arg is not None:
+ column = _string_content(column_arg)
+ return f'.in_("{column}", keys)' if column else None
+ if node.type == "comparison_operator" and any(c.type == "==" for c in node.children):
+ found = self._equality_filter(node, ref)
+ if found is not None:
+ return found
+ return None
+
+ def _eq_column_arg(self, call: Node, ref: Node, methods: set[str | None]) -> Node | None:
+ """The column string of supabase-py ``.table(t).select(...).eq("col", key)``."""
+ args = [c for c in call.child_by_field_name("arguments").children if c.is_named]
+ if (
+ len(args) == 2
+ and args[0].type == "string"
+ and self._is_key_of(args[1], ref)
+ and ("select" in methods) != ("delete" in methods)
+ ):
+ return args[0]
+ return None
+
+ def _equality_filter(self, comparison: Node, ref: Node) -> str | None:
+ """SQLAlchemy ``.where(M.col == key)`` / ``.filter(M.col == key)``."""
+ sides = [c for c in comparison.children if c.is_named]
+ if len(sides) != 2:
+ return None
+ for column, key in (sides, sides[::-1]):
+ path = self._dotted_path(column)
+ if path and "." in path and self._is_key_of(key, ref):
+ return f"{path}.in_(keys)"
+ return None
+
+ def concurrency_bound(self, sink: Node, loop: Node) -> str | None:
+ if self._exits_early(self.loop_body(loop) or loop):
+ return None
+ cur = sink.parent
+ while cur is not None and cur != loop:
+ if cur.type == "with_statement" and any(c.type == "async" for c in cur.children):
+ path = self._semaphore_entered(cur)
+ if path is not None:
+ return path
+ cur = cur.parent
+ return None
+
+ def _semaphore_entered(self, with_stmt: Node) -> str | None:
+ """The semaphore-like name an ``async with`` enters. A constructor
+ (``async with Semaphore(5)``) is fresh per iteration and bounds nothing."""
+ clause = next((c for c in with_stmt.children if c.type == "with_clause"), None)
+ for item in clause.children if clause is not None else ():
+ value = item.child_by_field_name("value") if item.type == "with_item" else None
+ path = self._dotted_path(value) if value is not None else None
+ if path and _PY_SEM_NAME_RE.search(path.rsplit(".", 1)[-1]):
+ return path
+ return None
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py
new file mode 100644
index 000000000..6e7d4b2c8
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py
@@ -0,0 +1,187 @@
+"""Python call, method and node-kind vocabularies the perf dialect matches on."""
+
+from __future__ import annotations
+
+import builtins
+import re
+
+# Network round-trip verbs (shared across languages that name HTTP methods).
+HTTP_VERBS: frozenset[str] = frozenset(
+ {"get", "post", "put", "delete", "patch", "head", "options", "request", "send", "stream"}
+)
+
+# Python DBAPI / SQLAlchemy execution sinks, split into three strata so the
+# ambiguous accessors can be gated harder than the unambiguous ones (see
+# ``sink_kind``).
+PY_DB_UNAMBIGUOUS: frozenset[str] = frozenset({"fetchone", "fetchall", "fetchmany"})
+# DB verbs other clients spell too: GitPython's ``repo.commit()``, an HTTP or
+# GraphQL SDK's ``request.execute()``, a builder's ``.scalar()``.
+PY_DB_NEEDS_EVIDENCE: frozenset[str] = frozenset(
+ {"commit", "execute", "executemany", "scalars", "scalar", "scalar_one", "scalar_one_or_none"}
+)
+# Without a db import, the call itself can still show it runs a query: a
+# PostgREST chain (``client.table("t")...execute()``) or a statement argument
+# (``session.execute(select(...))``, ``cur.execute("SELECT ...")``).
+_PY_QUERY_CHAIN_METHODS: frozenset[str] = frozenset({"table", "from_"})
+_PY_STATEMENT_BUILDERS: frozenset[str] = frozenset({"select", "insert", "update", "delete", "text"})
+_PY_SQL_LITERAL_RE = re.compile(
+ r"^[rRuUbBfF]*[\"']+\s*(?:SELECT|INSERT|UPDATE|DELETE|WITH|CREATE|DROP|ALTER|PRAGMA)\b",
+ re.IGNORECASE,
+)
+PY_DB_AMBIGUOUS: frozenset[str] = frozenset({"all", "first", "one", "one_or_none"})
+PY_SUBPROC_METHODS: frozenset[str] = frozenset(
+ {"run", "call", "check_call", "check_output", "Popen"}
+)
+# Any of these anywhere in a read's call chain proves it is bounded, gating
+# ``unbounded_read_reduced_in_memory`` (analysis.health.perf.unbounded_reduction).
+PY_UNBOUNDED_READ_BOUND_METHODS: frozenset[str] = frozenset(
+ {"limit", "range", "single", "maybe_single", "first", "count", "head", "aggregate"}
+)
+
+# Matched on the method name alone: only ``pathlib.Path`` spells these, and the
+# receiver is usually a local (``src_path.read_text()``) that ``io_names`` cannot
+# classify. Metadata calls (``exists`` / ``stat`` / ``glob``) are cheap and left out.
+PY_FS_METHODS: frozenset[str] = frozenset(
+ {"read_text", "read_bytes", "write_text", "write_bytes"}
+)
+# ``os`` / ``shutil`` verbs, gated on the literal module root. Ceiling: the root is
+# the leftmost name of the whole chain, so ``os.path.join(a).verb()`` lands here too;
+# no verb a ``str`` also has (``replace``) may join. Upgrade: require two segments.
+PY_OS_FS_METHODS: frozenset[str] = frozenset(
+ {
+ "walk",
+ "listdir",
+ "scandir",
+ "remove",
+ "unlink",
+ "rename",
+ "makedirs",
+ "mkdir",
+ "rmdir",
+ "removedirs",
+ }
+)
+PY_SHUTIL_FS_METHODS: frozenset[str] = frozenset(
+ {
+ "copy",
+ "copy2",
+ "copyfile",
+ "copytree",
+ "move",
+ "rmtree",
+ "make_archive",
+ "unpack_archive",
+ }
+)
+
+# Python string-literal node kinds (f-strings parse as ``string`` too).
+_PY_STRING_KINDS: frozenset[str] = frozenset({"string", "concatenated_string"})
+_PY_AUG_ASSIGN_KINDS: frozenset[str] = frozenset({"augmented_assignment"})
+
+# Connection / client / pool constructors, as ``(root, method)`` on a distinctive I/O library ...
+_PY_RESOURCE_CTORS: frozenset[tuple[str, str]] = frozenset(
+ {
+ ("sqlite3", "connect"),
+ ("psycopg", "connect"),
+ ("psycopg2", "connect"),
+ ("pymysql", "connect"),
+ ("MySQLdb", "connect"),
+ ("aiosqlite", "connect"),
+ ("httpx", "Client"),
+ ("httpx", "AsyncClient"),
+ ("requests", "Session"),
+ ("aiohttp", "ClientSession"),
+ ("boto3", "client"),
+ ("boto3", "resource"),
+ ("redis", "Redis"),
+ ("redis", "StrictRedis"),
+ ("pymongo", "MongoClient"),
+ }
+)
+# ... plus a few constructors distinctive enough on their own (the rightmost
+# name), so an aliased ``from sqlalchemy import create_engine`` still resolves.
+_PY_RESOURCE_METHODS: frozenset[str] = frozenset({"create_engine", "MongoClient"})
+
+# Lock acquisition only; ``release`` would count the same critical section twice.
+_PY_LOCK_METHODS: frozenset[str] = frozenset({"acquire"})
+
+# RHS node kinds that prove a name is bound to a list (the membership gate).
+_PY_LIST_RHS_KINDS: frozenset[str] = frozenset({"list", "list_comprehension"})
+# Builtins whose call result is provably a list.
+_PY_LIST_BUILTINS: frozenset[str] = frozenset({"list", "sorted"})
+# RHS shapes of a set or dict: a name bound to one anywhere in the file leaves the membership gate.
+_PY_NONLIST_RHS_KINDS: frozenset[str] = frozenset(
+ {"set", "dictionary", "set_comprehension", "dictionary_comprehension"}
+)
+_PY_NONLIST_BUILTINS: frozenset[str] = frozenset({"set", "dict", "frozenset"})
+
+# Promotion facts (perf/loop_facts.py). Calls and attributes that read a query
+# result out without changing how many rows it holds.
+_PY_RESULT_PROJECTIONS: frozenset[str] = frozenset(
+ {"all", "scalars", "fetchall", "json", "data", "rows"}
+)
+# Raw SQL capped in its own text: ``sa.text("... LIMIT 1000")``.
+_PY_SQL_LIMIT_RE = re.compile(rb"(?is)\bselect\b.*\blimit\s+\d+\b")
+_PY_SQL_COMMENT_RE = re.compile(rb"--[^\n]*|/\*.*?\*/", re.S)
+# A per-key call that limits or orders its rows is not the read one IN query makes.
+_PY_PER_KEY_LIMITS: frozenset[str] = frozenset(
+ {
+ "limit", "single", "maybe_single", "first", "one", "scalar_one",
+ "scalar_one_or_none", "order", "order_by", "range", "offset",
+ }
+)
+_PY_ONE_ROW: frozenset[str] = _PY_PER_KEY_LIMITS | {"scalar"}
+_PY_READ_CAPS: frozenset[str] = _PY_ONE_ROW - {"order", "order_by", "offset"}
+_PY_WRITES: frozenset[str] = frozenset({"update", "insert", "upsert", "values"})
+# Keys a query yields (``.in_(select(...))``) are as many as that table's rows.
+_PY_KEY_READS: frozenset[str] = _PY_STATEMENT_BUILDERS | {
+ "subquery", "scalar_subquery", "query", "cte", "alias", "union", "union_all",
+ "execute", "table", "from_",
+}
+_PY_SCOPES: frozenset[str] = frozenset({"function_definition", "lambda", "class_definition"})
+_PY_BUILTINS: frozenset[str] = frozenset(dir(builtins))
+_PY_SEM_NAME_RE = re.compile(r"(?i)(sem|semaphore|limiter|limit)$")
+# A model class is CapWords; an ALL_CAPS constant (``API_URL``) is not.
+_PY_MODEL_NAME_RE = re.compile(r"^[A-Z]\w*[a-z]\w*$")
+# Directory listings; the recursive ones walk a whole tree.
+_PY_LISTINGS: frozenset[str] = frozenset(
+ {
+ "glob", "iglob", "rglob", "iterdir", "walk", "listdir", "scandir", "traverse",
+ "iter_glob", "glob_via", "walk_repo", "_rglob",
+ }
+)
+_PY_RECURSIVE_LISTINGS: frozenset[str] = frozenset(
+ {"rglob", "walk", "traverse", "walk_repo", "_rglob"}
+)
+# Reads that yield lines or chunks as they arrive.
+_PY_STREAMS: frozenset[str] = frozenset(
+ {
+ "readlines", "iter_lines", "iter_content", "iter_bytes", "iter_text",
+ "aiter_lines", "aiter_bytes", "aiter_text", "aiter_raw",
+ }
+)
+# Wrappers and chunkers whose first argument is what the loop walks.
+_PY_PASS_THROUGH: frozenset[str] = frozenset(
+ {
+ "enumerate", "sorted", "reversed", "list", "set", "tuple", "iter", "zip",
+ "batched", "chunked", "ichunked", "chunks", "iter_chunks", "grouper",
+ }
+)
+_PY_CONSTANT_RHS: frozenset[str] = frozenset({"list", "tuple", "set", "dictionary"})
+
+
+_PY_DEF_KINDS = frozenset({"function_definition", "lambda", "class_definition"})
+# Nodes an identifier sits in when it is the name being bound, not read.
+_PY_BINDING_PARENTS = frozenset(
+ {
+ "global_statement", "nonlocal_statement", "as_pattern_target", "aliased_import",
+ "import_from_statement", "import_statement", "parameters", "lambda_parameters",
+ "named_expression",
+ }
+)
+_PY_BINDING_WRAPPERS = frozenset(
+ {
+ "pattern_list", "tuple_pattern", "list_pattern", "typed_parameter", "default_parameter",
+ "typed_default_parameter", "list_splat_pattern", "dotted_name",
+ }
+)
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py
new file mode 100644
index 000000000..9d51e6da2
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py
@@ -0,0 +1,193 @@
+"""Tree-sitter helpers for Python syntax the perf dialect reads."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .base import BasePerfDialect
+from .python_lexicon import (
+ _PY_BINDING_PARENTS,
+ _PY_BINDING_WRAPPERS,
+ _PY_CONSTANT_RHS,
+ _PY_DEF_KINDS,
+)
+
+if TYPE_CHECKING:
+ from tree_sitter import Node
+
+
+def _binds(ident: Node) -> bool:
+ """Whether *ident* is a name being bound (assigned, imported, a parameter)."""
+ cur, parent = ident, ident.parent
+ while parent is not None and parent.type in _PY_BINDING_WRAPPERS:
+ cur, parent = parent, parent.parent
+ if parent is None:
+ return False
+ if parent.type in ("assignment", "augmented_assignment", "for_statement", "for_in_clause"):
+ return parent.child_by_field_name("left") == cur
+ if parent.type in ("function_definition", "class_definition"):
+ return parent.child_by_field_name("name") == cur
+ return parent.type in _PY_BINDING_PARENTS
+
+
+def _bindings(scope: Node, name: bytes) -> list[Node]:
+ """Identifiers binding *name* in *scope* itself, not in a nested scope."""
+ found: list[Node] = []
+ stack = list(scope.children)
+ while stack:
+ node = stack.pop()
+ if node.type in _PY_DEF_KINDS:
+ stack.extend(c for c in (node.child_by_field_name("name"),) if c is not None)
+ continue
+ if node.type == "identifier" and node.text == name and _binds(node):
+ found.append(node)
+ stack.extend(node.children)
+ return found
+
+
+def _module_int_constant(name: Node) -> bool:
+ """An ALL_CAPS name its own module binds exactly once, to an integer literal
+ (``MAX = 5``), and no enclosing function rebinds or shadows.
+
+ The spelling alone proves nothing (``NUM_USERS`` grows), so a name rebound,
+ imported, augmented, written through ``global``, shadowed by a parameter or
+ a local, or bound to anything but a literal stays unknown.
+ """
+ text = name.text or b""
+ if name.type != "identifier" or len(text) < 2 or not text.decode().isupper():
+ return False
+ module = _unshadowed_module(name, text)
+ if module is None:
+ return False
+ bound = _bindings(module, text)
+ if len(bound) != 1 or bound[0].parent is None or bound[0].parent.type != "assignment":
+ return False
+ right = bound[0].parent.child_by_field_name("right")
+ return right is not None and right.type == "integer"
+
+
+def _unshadowed_module(name: Node, text: bytes) -> Node | None:
+ """The module root, when no enclosing function binds *text* and no ``global`` names it."""
+ scope = name.parent
+ while scope is not None and scope.parent is not None:
+ if scope.type in ("function_definition", "lambda") and _bindings(scope, text):
+ return None
+ scope = scope.parent
+ if scope is None or any(
+ n.type == "global_statement" and any(c.text == text for c in n.named_children)
+ for n in BasePerfDialect._walk(scope)
+ ):
+ return None
+ return scope
+
+
+def _int_literal(node: Node) -> bool:
+ return node.type == "integer" or (
+ node.type == "unary_operator" and any(c.type == "integer" for c in node.children)
+ )
+
+
+def _literal_range(call: Node) -> bool:
+ """``range(...)`` whose every argument is an integer literal; ``range(len(x))`` is data-dependent."""
+ fn = call.child_by_field_name("function")
+ if fn is None or (fn.text or b"").decode("utf-8", "replace") != "range":
+ return False
+ args = call.child_by_field_name("arguments")
+ if args is None:
+ return False
+ return all(_int_literal(a) for a in args.children if a.is_named)
+
+
+def _is_or(expr: Node) -> bool:
+ return expr.type == "boolean_operator" and any(c.type == "or" for c in expr.children)
+
+
+def _slice_bounds(subscript: Node) -> tuple[Node | None, Node] | None:
+ """``(start, stop)`` of a one-colon slice that has a stop."""
+ piece = subscript.child_by_field_name("subscript")
+ if piece is None or piece.type != "slice" or sum(c.type == ":" for c in piece.children) != 1:
+ return None
+ colon = next(i for i, c in enumerate(piece.children) if c.type == ":")
+ start = next((c for c in piece.children[:colon] if c.is_named), None)
+ stop = next((c for c in piece.children[colon + 1 :] if c.is_named), None)
+ return None if stop is None else (start, stop)
+
+
+def _string_content(string: Node) -> str | None:
+ return next((c.text.decode() for c in string.children if c.type == "string_content"), None)
+
+
+def _text(node: Node) -> str:
+ return (node.text or b"").decode("utf-8", "replace")
+
+
+def _field_text(node: Node, field: str) -> str:
+ child = node.child_by_field_name(field)
+ return _text(child) if child is not None else ""
+
+
+def _value_refs(expr: Node) -> list[Node]:
+ """The names and dotted paths *expr* reads values from: not a callee
+ (``tuple`` in ``tuple(xs)``), a keyword, or an ALL_CAPS constant."""
+ refs = []
+ stack = [expr]
+ while stack:
+ node = stack.pop()
+ if node.type not in ("identifier", "attribute"):
+ stack.extend(node.children)
+ continue
+ parent = node.parent
+ callee = parent is not None and parent.child_by_field_name("function") == node
+ keyword = parent is not None and parent.type == "keyword_argument" and (
+ parent.child_by_field_name("name") == node
+ )
+ last = node.child_by_field_name("attribute") if node.type == "attribute" else node
+ constant = last is not None and _text(last).strip("_").isupper()
+ if callee and node.type == "attribute":
+ # ``self.ids.copy()`` still reads ``self.ids``.
+ stack.extend(filter(None, [node.child_by_field_name("object")]))
+ elif not (callee or keyword or constant):
+ refs.append(node)
+ return refs
+
+
+def _string_value(node: Node) -> str | None:
+ return _text(node).strip("\"'") if node.type == "string" else None
+
+
+def _in_filter(call: Node, args: list[Node]) -> tuple[str, Node] | None:
+ """``(column, keys)`` of ``Model.col.in_(keys)`` or PostgREST ``.in_("col", keys)``."""
+ if len(args) == 2:
+ column = _string_value(args[0])
+ return (column, args[1]) if column else None
+ fn = call.child_by_field_name("function")
+ receiver = fn.child_by_field_name("object") if fn is not None else None
+ if len(args) != 1 or receiver is None or receiver.type != "attribute":
+ return None
+ column = _field_text(receiver, "attribute")
+ return (column, args[0]) if column else None
+
+
+def _fixed_set(value: Node | None) -> bool:
+ """A non-empty literal of scalars; an empty one is a container filled later."""
+ return (
+ value is not None
+ and value.type in _PY_CONSTANT_RHS
+ and bool(value.named_children)
+ and all(_scalar_literal(item) for item in value.named_children)
+ )
+
+
+_PY_SCALARS = frozenset({"string", "concatenated_string", "integer", "float", "true", "false", "none"})
+
+
+def _scalar_literal(item: Node) -> bool:
+ """``"a"`` / ``3`` / ``("a", 1)`` / ``"k": v`` by its key: an entry the source
+ spells out. A name or call in the literal (``[ROOT / "a", repo.path]``) may
+ stand for anything, so the table is not shown to be a fixed set."""
+ if item.type == "pair":
+ key = item.child_by_field_name("key")
+ return key is not None and key.type in _PY_SCALARS
+ if item.type == "tuple":
+ return all(child.type in _PY_SCALARS for child in item.named_children)
+ return item.type in _PY_SCALARS
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_sources.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_sources.py
new file mode 100644
index 000000000..7a947e9a7
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_sources.py
@@ -0,0 +1,350 @@
+"""What a Python loop walks, and how large it can grow."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .. import loop_lexicon as lex
+from ..loop_facts import LoopSource
+from .python_lexicon import (
+ _PY_CONSTANT_RHS,
+ _PY_KEY_READS,
+ _PY_LISTINGS,
+ _PY_MODEL_NAME_RE,
+ _PY_PASS_THROUGH,
+ _PY_READ_CAPS,
+ _PY_RESULT_PROJECTIONS,
+ _PY_SQL_COMMENT_RE,
+ _PY_SQL_LIMIT_RE,
+ _PY_SQL_LITERAL_RE,
+ _PY_STREAMS,
+)
+from .python_nodes import (
+ _field_text,
+ _fixed_set,
+ _in_filter,
+ _is_or,
+ _module_int_constant,
+ _scalar_literal,
+ _slice_bounds,
+ _string_value,
+ _text,
+ _value_refs,
+)
+
+if TYPE_CHECKING:
+ from tree_sitter import Node
+
+ from ..loop_facts import LoopIterates, LoopMagnitude, SinkProbe
+
+ Source = tuple[LoopIterates, LoopMagnitude | None]
+
+
+class PythonLoopSources:
+ """Loop sources and magnitudes. Mixed into :class:`~.python.PythonPerfDialect`."""
+
+ def loop_source(self, loop: Node, probe: SinkProbe) -> LoopSource | None:
+ if loop.type != "for_statement":
+ return None
+ right = loop.child_by_field_name("right")
+ baseline = self._magnitude(right, loop, probe, hop=True) or "unknown"
+ iterates, claim = self._source(right, loop, probe, hop=True)
+ if iterates == "unknown" and baseline == "unknown":
+ return None
+ return LoopSource(iterates, claim or baseline)
+
+ def _source(self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ """What *expr* walks, and the magnitude that settles when it differs from
+ what the iterable's syntax alone proves (``None``: keep that).
+
+ *hop* allows one step through the name's last assignment before the loop.
+ """
+ if expr is None:
+ return "unknown", None
+ if expr.type in ("await", "parenthesized_expression"):
+ inner = next((c for c in expr.children if c.is_named), None)
+ return self._source(inner, loop, probe, hop=hop)
+ if _is_or(expr):
+ return self._source(expr.child_by_field_name("left"), loop, probe, hop=hop)
+ if expr.type in ("identifier", "attribute"):
+ return self._named_source(expr, loop, probe, hop=hop)
+ if expr.type == "subscript" and self._slice_magnitude(expr) == "bounded":
+ # A fixed-width slice bounds the trip count, not what is walked.
+ walked = self._source(expr.child_by_field_name("value"), loop, probe, hop=hop)
+ return walked[0], "bounded"
+ if expr.type == "call":
+ return self._call_source(expr, loop, probe, hop=hop)
+ return "unknown", None
+
+ def _named_source(self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ if lex.is_config_name(_text(expr)):
+ # A registry grows with what users register: the name alone bounds nothing.
+ return "config_table", None
+ if expr.type == "identifier":
+ return self._name_source(expr, loop, probe, hop=hop)
+ return self._attribute_source(expr, loop, probe, hop=hop)
+
+ def _attribute_source(self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ """``result.rows`` walks ``result``; ``self._TABLES`` bound to a literal is a constant."""
+ attr = expr.child_by_field_name("attribute")
+ name = (attr.text if attr is not None else b"") or b""
+ if name.decode("utf-8", "replace") in _PY_RESULT_PROJECTIONS:
+ return self._source(expr.child_by_field_name("object"), loop, probe, hop=hop)
+ if self._literal_constant(expr, name):
+ return "constant", "bounded"
+ return "unknown", None
+
+ def _name_source(self, name: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ rhs = self.reaching_value(loop, name.text or b"") if hop else None
+ if rhs is not None:
+ return self._source(rhs, loop, probe, hop=False)
+ fn = loop.parent
+ while fn is not None and fn.type != "function_definition":
+ fn = fn.parent
+ params = fn.child_by_field_name("parameters") if fn is not None else None
+ if params is not None and _text(name) in self._param_names(params):
+ return "parameter", None
+ return "unknown", None
+
+ def _call_source(self, call: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ method, fn, receiver, args = self._call_bits(call)
+ if lex.is_config_name(self._dotted_path(fn) or "", callee=True):
+ return "config_table", None
+ if method == "range" and receiver is None:
+ return self._range_source(call, args, loop, probe, hop=hop)
+ walked = self._walked_arg_or_receiver(method, receiver, args)
+ if walked is not None:
+ return self._source(walked, loop, probe, hop=hop)
+ if method in _PY_LISTINGS or method in _PY_STREAMS or (method == "open" and not receiver):
+ return self._file_source(call, method, loop)
+ if probe(call) in ("db", "network"):
+ return self._query(call, loop)
+ if receiver is not None and method in _PY_RESULT_PROJECTIONS:
+ return self._source(receiver, loop, probe, hop=hop)
+ return "unknown", None
+
+ def _call_bits(self, call: Node) -> tuple[str, Node | None, Node | None, list[Node]]:
+ """``(method, callee, receiver, positional args)`` of a call."""
+ fn = call.child_by_field_name("function")
+ receiver = fn.child_by_field_name("object") if fn is not None and fn.type == "attribute" else None
+ arg_list = call.child_by_field_name("arguments")
+ args = [c for c in arg_list.children if c.is_named] if arg_list is not None else []
+ return self.callee_method_name(call) or "", fn, receiver, args
+
+ @staticmethod
+ def _walked_arg_or_receiver(
+ method: str, receiver: Node | None, args: list[Node]
+ ) -> Node | None:
+ """``sorted(xs)`` / ``enumerate(xs)`` walk ``xs``; ``d.items()`` walks ``d``."""
+ if method in _PY_PASS_THROUGH and args and args[0].type != "keyword_argument":
+ return args[0]
+ if receiver is not None and method in ("items", "values", "keys"):
+ return receiver
+ return None
+
+ def _file_source(self, call: Node, method: str, loop: Node) -> Source:
+ """A directory listing (grows when it walks the repository tree) or a stream."""
+ if method in _PY_LISTINGS:
+ return "listing", "grows_with_data" if self._walks_repo(call, method, loop) else None
+ return "stream", None
+
+ def _range_source(
+ self, call: Node, args: list[Node], loop: Node, probe: SinkProbe, *, hop: bool
+ ) -> Source:
+ """``range(3)`` is a constant; ``range(0, len(ids), N)`` walks ``ids``."""
+ if self._range_magnitude(call, loop, probe, hop=hop) == "bounded":
+ return "constant", "bounded"
+ stop = args[1] if len(args) > 1 else (args[0] if args else None)
+ if stop is not None and stop.type == "call" and self.callee_method_name(stop) == "len":
+ inner = [c for c in stop.child_by_field_name("arguments").children if c.is_named]
+ if len(inner) == 1:
+ return self._source(inner[0], loop, probe, hop=hop)
+ return "unknown", None
+
+ def reads_config_table(self, call: Node, nodes: list[Node]) -> bool:
+ """Every SQL text and selected model names a registry or configuration table."""
+ walked = [n for part in nodes for n in self._walk(part)]
+ sql = [
+ n.text or b""
+ for n in walked
+ if n.type == "string" and _PY_SQL_LITERAL_RE.match(_text(n))
+ ]
+ models = self._query_models(walked)
+ return bool(sql or models) and (
+ all(lex.is_config_sql(text) for text in sql)
+ and all(lex.is_config_model(model) for model in models)
+ )
+
+ def _query_models(self, walked: list[Node]) -> list[str]:
+ return [
+ model
+ for n in walked
+ if n.type == "call" and self.callee_method_name(n) in ("select", "query")
+ for model in self._selected_models(n)
+ ]
+
+ def _selected_models(self, select: Node) -> list[str]:
+ """``select(Repository)`` / ``query(Repository.id, Repository.path)`` -> the model
+ names; ``func.count()`` and other expressions name none."""
+ args = select.child_by_field_name("arguments")
+ models: list[str] = []
+ for arg in (c for c in args.children if c.is_named) if args is not None else ():
+ path = self._dotted_path(arg) or ""
+ head = next((seg for seg in path.split(".") if _PY_MODEL_NAME_RE.match(seg)), None)
+ if head is not None:
+ models.append(head)
+ return models
+
+ def _literal_constant(self, expr: Node, name: bytes) -> bool:
+ """``self._TABLES`` where the file binds ``_TABLES`` once, to a literal collection."""
+ text = name.decode("utf-8", "replace")
+ if len(text.strip("_")) < 2 or not text.isupper():
+ return False
+ root = expr
+ while root.parent is not None:
+ root = root.parent
+ rights = [
+ n.child_by_field_name("right")
+ for n in self._walk(root)
+ if n.type == "assignment"
+ and (left := n.child_by_field_name("left")) is not None
+ and left.type == "identifier"
+ and left.text == name
+ ]
+ return (
+ len(rights) == 1
+ and rights[0] is not None
+ and rights[0].type in _PY_CONSTANT_RHS
+ and all(_scalar_literal(item) for item in rights[0].named_children)
+ )
+
+ def _magnitude(
+ self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool
+ ) -> LoopMagnitude | None:
+ """Grows (a query result or a directory listing), bounded, or ``None``.
+
+ *hop* allows one step through the name's last assignment before the
+ loop; a parameter has none and stays unknown on purpose.
+ """
+ if expr is None:
+ return None
+ if expr.type in ("await", "parenthesized_expression"):
+ inner = next((c for c in expr.children if c.is_named), None)
+ return self._magnitude(inner, loop, probe, hop=hop)
+ if expr.type == "identifier" and hop and expr.text:
+ rhs = self.reaching_value(loop, expr.text)
+ return self._magnitude(rhs, loop, probe, hop=False)
+ if expr.type == "subscript":
+ return self._slice_magnitude(expr)
+ if _is_or(expr):
+ # ``(await q.execute()).data or []``: the fallback is empty, the read is not.
+ left = expr.child_by_field_name("left")
+ return self._grows(self._magnitude(left, loop, probe, hop=hop))
+ if expr.type == "attribute":
+ return self._attribute_magnitude(expr, loop, probe, hop=hop)
+ return self._call_magnitude(expr, loop, probe, hop=hop) if expr.type == "call" else None
+
+ def _attribute_magnitude(
+ self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool
+ ) -> LoopMagnitude | None:
+ attr = expr.child_by_field_name("attribute")
+ if attr is None or (attr.text or b"").decode() not in _PY_RESULT_PROJECTIONS:
+ return None
+ return self._grows(self._magnitude(expr.child_by_field_name("object"), loop, probe, hop=hop))
+
+ def _call_magnitude(
+ self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool
+ ) -> LoopMagnitude | None:
+ method = self.callee_method_name(expr) or ""
+ member = self.callee_is_attribute(expr)
+ if method == "range" and not member:
+ return self._range_magnitude(expr, loop, probe, hop=hop)
+ if probe(expr) in ("db", "network"):
+ # A read capped in the query is as large as its cap; a statement or result
+ # built before the loop (``stmt = ....limit(n)``) is read one assignment back.
+ names = [n.text for n in self._walk(expr) if n.type == "identifier"]
+ parts = [expr, *filter(None, (self.reaching_value(loop, name) for name in names))]
+ capped = any(self.is_cap(n, loop) for part in parts for n in self._walk(part))
+ return None if capped else "grows_with_data"
+ if member and method in _PY_RESULT_PROJECTIONS:
+ receiver = expr.child_by_field_name("function").child_by_field_name("object")
+ return self._grows(self._magnitude(receiver, loop, probe, hop=hop))
+ return None
+
+ def is_cap(self, node: Node, loop: Node) -> bool:
+ """``.limit(n)`` and peers, a SQL string saying ``LIMIT n``, or a filter to a
+ set of keys (:meth:`read_key_column`), which is as large as the keys are."""
+ if node.type == "string":
+ return bool(_PY_SQL_LIMIT_RE.search(_PY_SQL_COMMENT_RE.sub(b"", node.text or b"")))
+ method = self.callee_method_name(node) if node.type == "call" else None
+ if method in _PY_READ_CAPS:
+ return True
+ return node.type == "call" and self.read_key_column(node, loop) is not None
+
+ def read_key_column(self, call: Node, anchor: Node) -> str | None:
+ """``Model.col.in_(keys)`` and PostgREST ``.in_("col", keys)`` -> ``col``;
+ ``filter_by(id=k)`` and ``.eq("id", k)`` -> ``id``. Keys a query yields
+ (``.in_(select(...))``, or a name bound to one) grow with that table."""
+ method = self.callee_method_name(call)
+ arg_list = call.child_by_field_name("arguments")
+ args = arg_list.named_children if arg_list is not None else []
+ if method == "filter_by":
+ named = {_field_text(a, "name") for a in args if a.type == "keyword_argument"}
+ return "id" if "id" in named else None
+ if method == "eq":
+ return "id" if len(args) == 2 and _string_value(args[0]) == "id" else None
+ found = _in_filter(call, args) if method == "in_" else None
+ if found is None or not self._handed_in(found[1], anchor):
+ return None
+ return found[0]
+
+ def _handed_in(self, keys: Node, anchor: Node) -> bool:
+ """*keys* holds values given to the code, not ones a query yields or a fixed
+ set of states (``in_(("a", "b"))``, ``in_(SOURCES)``) whose rows grow."""
+ values = [
+ self.reaching_value(anchor, ref.text or b"") if ref.type == "identifier" else None
+ for ref in _value_refs(keys)
+ ]
+ if all(_fixed_set(value) for value in values):
+ return False
+ parts = [keys, *filter(None, values)]
+ return not any(
+ n.type == "call" and self.callee_method_name(n) in _PY_KEY_READS
+ for part in parts
+ for n in self._walk(part)
+ )
+
+ def _range_magnitude(
+ self, call: Node, loop: Node, probe: SinkProbe, *, hop: bool
+ ) -> LoopMagnitude | None:
+ """``range(len(rows))`` grows with ``rows``; ``range(3)`` and ``range(MAX)``
+ with ``MAX = 3`` bound once in the module are bounded."""
+ args = [c for c in call.child_by_field_name("arguments").children if c.is_named]
+ if not args:
+ return None
+ stop = args[1] if len(args) > 1 else args[0]
+ if stop.type == "call" and self.callee_method_name(stop) == "len":
+ inner = [c for c in stop.child_by_field_name("arguments").children if c.is_named]
+ if len(inner) == 1:
+ return self._grows(self._magnitude(inner[0], loop, probe, hop=hop))
+ return None
+ # A name alone (``MAX_RETRIES``, ``self.max_attempts``) proves nothing.
+ return "bounded" if stop.type == "integer" or _module_int_constant(stop) else None
+
+ def _slice_magnitude(self, subscript: Node) -> LoopMagnitude | None:
+ """``xs[:5]`` / ``xs[:LIMIT]`` / ``xs[i:i + 4]``: a constant-width slice."""
+ bounds = _slice_bounds(subscript)
+ if bounds is None:
+ return None
+ start, stop = bounds
+ if start is None:
+ return "bounded" if stop.type == "integer" or _module_int_constant(stop) else None
+ return "bounded" if self._offset_by_int(start, stop) else None
+
+ def _offset_by_int(self, start: Node, stop: Node) -> bool:
+ """*stop* is ``start + <integer>``."""
+ if stop.type != "binary_operator":
+ return False
+ left, right = stop.child_by_field_name("left"), stop.child_by_field_name("right")
+ same_start = left is not None and self._dotted_path(left) == self._dotted_path(start)
+ return same_start and right is not None and right.type == "integer"