Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(perf): say whether Go, Java, C# and Ruby perf loops grow with the data (#3489)

Raghav Chamadiya4756386e9bae
diff --git a/docs/layers/PERFORMANCE.md b/docs/layers/PERFORMANCE.md
index 0430f43fb..c2ee6f5bd 100644
--- a/docs/layers/PERFORMANCE.md
+++ b/docs/layers/PERFORMANCE.md
@@ -57,6 +57,14 @@ that sink comes first. The plan also lists tests to run, found from coverage, th
graph and the import graph. An opportunity with a supported shared fix links to a
`performance_fix` plan in [REFACTORING.md](REFACTORING.md).
+Each loop also says what it walks: rows a query returns, a collection its caller
+passes in, the rows of one key, a registry or configuration table, a fixed set, a
+directory listing or a stream, and plan steps name it. Python, TypeScript/JavaScript,
+Go, Java, C# and Ruby loops say whether they grow with the data or are bounded; a
+loop whose size the code does not show stays unknown and is never treated as
+bounded. A loop that stops at the first call that answers, or feeds each call's
+result into the next, gets no batch advice (reason `loop_not_batchable`).
+
## The default queue
The default queue holds production opportunities in the `plan_ready` and
diff --git a/packages/core/src/repowise/core/analysis/change_health/service.py b/packages/core/src/repowise/core/analysis/change_health/service.py
index 682114057..e215b5a9a 100644
--- a/packages/core/src/repowise/core/analysis/change_health/service.py
+++ b/packages/core/src/repowise/core/analysis/change_health/service.py
@@ -487,6 +487,8 @@ _INHERENT = "Nothing to change: the repetition is inherent or already batched."
_EXPECTED_TEXT: dict[str | None, str] = {
"inherent_to_boundary": _INHERENT,
"loop_already_chunked": _INHERENT,
+ "loop_not_batchable": "Nothing to batch: each call needs the one before, or the loop "
+ "stops at the first that answers.",
"gated_off": "Nothing to change while a constant-False flag switches this function off.",
"cold_path": "Runs once per deploy, boot or incident; not worth scheduling.",
"bounded_loop": "Nothing to change: the loop runs a fixed number of times.",
diff --git a/packages/core/src/repowise/core/analysis/health/complexity/perf_walk.py b/packages/core/src/repowise/core/analysis/health/complexity/perf_walk.py
index 89e25efcd..44f298978 100644
--- a/packages/core/src/repowise/core/analysis/health/complexity/perf_walk.py
+++ b/packages/core/src/repowise/core/analysis/health/complexity/perf_walk.py
@@ -26,7 +26,7 @@ from typing import TYPE_CHECKING
from ..perf.dialects import PERF_DIALECTS
from ..perf.dialects.base import BasePerfDialect as BasePerfDialectClass
-from ..perf.loop_facts import LoopFacts
+from ..perf.loop_facts import LoopFacts, LoopSource, combined_magnitude
from .ast_utils import (
_dart_signature_sibling,
_find_function_entry_name,
@@ -255,15 +255,25 @@ def _is_lock_acquire_spin(
def _enclosing_loops(
- node: Node, dialect: BasePerfDialect, loop_kinds: frozenset[str], fn_kinds: frozenset[str]
+ node: Node,
+ dialect: BasePerfDialect,
+ loop_kinds: frozenset[str],
+ fn_kinds: frozenset[str],
+ block_loops: bool = False,
) -> list[Node]:
"""The data-dependent loops around *node* in its function, innermost first;
- walked up only for hits. A loop whose iterable holds *node* is not one."""
+ walked up only for hits. A loop whose iterable holds *node* is not one. With
+ *block_loops*, a block-iteration call (Ruby ``items.each do``) whose block
+ holds *node* is one."""
loops: list[Node] = []
cur = node.parent
for _ in range(64):
if cur is None or cur.type in fn_kinds:
break
+ if block_loops and not dialect.is_constant_loop(cur):
+ block = dialect.block_loop_body(cur)
+ if block is not None and block.start_byte <= node.start_byte < block.end_byte:
+ loops.append(cur)
if cur.type in loop_kinds and cur.is_named and not dialect.is_constant_loop(cur):
iterable = dialect.iterable_node(cur)
# ``for row in result.all():`` runs ``.all()`` once per pass of the loop
@@ -526,7 +536,7 @@ def _collect_perf_hits(
and _overrides(dialect, "bare_statement_call")
)
do_chunked = _overrides(dialect, "is_chunked_loop")
- do_magnitude = _overrides(dialect, "loop_magnitude")
+ do_magnitude = _overrides(dialect, "loop_source")
do_batch = _overrides(dialect, "batch_form")
do_bound = _overrides(dialect, "concurrency_bound")
do_loop_facts = do_chunked or do_magnitude or do_batch or do_bound
@@ -538,13 +548,14 @@ def _collect_perf_hits(
call, awaited=dialect.is_awaited(call), io_names=io_names, has_db_import=has_db_import
)
- def magnitude(loops: list[Node]) -> str:
- """Grows if any enclosing loop grows; bounded only if every one is, since a
- retry loop inside a loop over rows still runs once per row."""
- found = {dialect.loop_magnitude(loop, probe) or "unknown" for loop in loops}
- if "grows_with_data" in found:
- return "grows_with_data"
- return "bounded" if found == {"bounded"} else "unknown"
+ # One answer per loop, however many hits sit in it.
+ sources: dict[tuple[int, int], LoopSource] = {}
+
+ def source(loop: Node) -> LoopSource:
+ key = (loop.start_byte, loop.end_byte)
+ if key not in sources:
+ sources[key] = dialect.loop_source(loop, probe) or LoopSource()
+ return sources[key]
def loop_facts(node: Node, sink: bool, per_call: bool = False) -> LoopFacts | None:
"""Facts of *node*'s innermost loop; batch and bound only mean something at a sink.
@@ -554,16 +565,22 @@ def _collect_perf_hits(
"""
if not (do_loop_facts or per_call):
return None
- loops = _enclosing_loops(node, dialect, loop_kinds, fn_kinds)
+ loops = _enclosing_loops(node, dialect, loop_kinds, fn_kinds, do_block_loop)
if not loops:
return None
loop = loops[0]
facts = LoopFacts(
chunked=do_chunked and dialect.is_chunked_loop(loop),
- magnitude=magnitude(loops) if do_magnitude else "unknown",
+ magnitude=(
+ combined_magnitude([source(lp) for lp in loops]) if do_magnitude else "unknown"
+ ),
batch=dialect.batch_form(node, loop, probe) if sink and do_batch else None,
concurrency_bound=dialect.concurrency_bound(node, loop) if sink and do_bound else None,
- key_unused=per_call and not _loop_key_feeds(node, loop),
+ # Only a native loop header names its key; a block loop (Ruby
+ # ``each do |row|``) claims nothing, as before it carried loop facts.
+ key_unused=per_call and loop.type in loop_kinds and not _loop_key_feeds(node, loop),
+ iterates=source(loop).iterates if do_magnitude else "unknown",
+ not_batchable=dialect.not_batchable(node, loop, loop_kinds) if per_call else None,
)
return facts if facts != LoopFacts() else None
diff --git a/packages/core/src/repowise/core/analysis/health/perf/actionability.py b/packages/core/src/repowise/core/analysis/health/perf/actionability.py
index b2d506c9d..78c2045fc 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/actionability.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/actionability.py
@@ -37,7 +37,9 @@ ActionabilityState = Literal["plan_ready", "advisory", "investigate", "expected"
EXPECTED_REASONS: tuple[str, ...] = ("gated_off", "cold_path")
# Refusals that are facts about the code, not missing proofs: nothing to investigate.
-_EXPECTED_REFUSALS = frozenset({"inherent_to_boundary", "loop_already_chunked"})
+_EXPECTED_REFUSALS = frozenset(
+ {"inherent_to_boundary", "loop_already_chunked", "loop_not_batchable"}
+)
FixStrategy = Literal[
"parallelize_independent_awaits",
"replace_membership_collection",
@@ -244,6 +246,10 @@ def assess_fix(
# Filesystem and subprocess repetition is real, but there is no
# batch or prefetch operation to point the caller at.
return FixAssessment(None, (), refusal="inherent_to_boundary")
+ if details and all(detail.get("loop_not_batchable") for detail in details):
+ # The loop stops at the first call that answers, or each call needs the
+ # one before it: there is no set of keys to send at once.
+ return FixAssessment(None, (), refusal="loop_not_batchable")
form = _shared(details, "batch_form")
if form:
if _shared(details, "batch_equivalent") is True:
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py
index c750bf6e7..744741739 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py
@@ -78,7 +78,7 @@ if TYPE_CHECKING:
from tree_sitter import Node
- from ..loop_facts import BatchForm, LoopMagnitude, SinkProbe
+ from ..loop_facts import BatchForm, LoopMagnitude, LoopSource, NotBatchable, SinkProbe
# Node types whose callee is a member access (``x.foo()``) rather than a bare
# identifier call. Covers Python ``attribute``, TS ``member_expression``, C++/
@@ -91,6 +91,11 @@ _ATTRIBUTE_CALLEE_KINDS: frozenset[str] = frozenset(
)
+# A result name that holds an error (``err``, ``fetch_error``): a loop that
+# exits on it stops at the first failure, which is no first match.
+_ERROR_NAME = re.compile(rb"(?:^|_)(?:err|error|exc|exception)$|(?:Err|Error)$")
+
+
class BasePerfDialect:
"""Default ``PerfDialect`` implementation — every method is "no signal".
@@ -374,10 +379,149 @@ class BasePerfDialect:
# Methods that grow a sequence in call order (``append`` / ``push``).
sequence_appends: frozenset[str] = frozenset()
+ def loop_source(self, loop: Node, probe: SinkProbe) -> LoopSource | None:
+ """What the loop's iterable is and whether it grows with data or has a
+ fixed small bound, or ``None`` when the syntax settles neither."""
+ return None
+
def loop_magnitude(self, loop: Node, probe: SinkProbe) -> LoopMagnitude | None:
- """Whether the loop's iterable grows with data or has a fixed small bound."""
+ source = self.loop_source(loop, probe)
+ return source.magnitude if source is not None else None
+
+ # The exits that end the loop early (``continue`` and a raise do not stop at a hit).
+ _STOPPING_EXITS = frozenset({"break_statement", "return_statement"})
+
+ def not_batchable(
+ self, call: Node, loop: Node, loop_kinds: frozenset[str]
+ ) -> NotBatchable | None:
+ """Why *call*, made once per pass of *loop*, cannot become one batch.
+
+ ``first_match``: an exit that ends the loop reads the call's result, so
+ later iterations run only while earlier calls miss. ``turn_loop``: the
+ call (or the loop's condition) reads a name the body binds from an
+ earlier call's result. ``None`` when neither is shown; a dialect that
+ names no binding statements never claims either.
+ """
+ if not self.binding_fields:
+ return None
+ body = self.loop_body(loop) or loop
+ # Only the call's own result says it answered; a local derived from it
+ # (``ok = check(r)``) may test something else, and an error it returned
+ # (``if err != nil { return err }``) stops at a failure, not a match.
+ answered = {n for n in self._result_names(call, body) if not _ERROR_NAME.search(n)}
+ if self._stops_at_hit(call, loop, body, answered, loop_kinds):
+ return "first_match"
+ results = self._result_names(call, body, hops=2)
+ # A sequence the results are appended to carries them into the next turn
+ # (``messages.append(reply)``); capping it (``len(out) >= n``) is no hit.
+ results |= self._appended_to(call, body, results)
+ condition = loop.child_by_field_name("condition")
+ carried = self._reads(call, results) or (
+ condition is not None and self._reads(condition, results)
+ )
+ return "turn_loop" if carried else None
+
+ def _bound_by(self, node: Node) -> tuple[Node | None, Node | None]:
+ """``(target, value)`` of a binding statement that is not a loop header."""
+ field = self.binding_fields.get(node.type)
+ if field is None or "for" in node.type:
+ return None, None
+ value = node.child_by_field_name("right") or node.child_by_field_name("value")
+ return node.child_by_field_name(field), value
+
+ def _texts(self, target: Node) -> set[bytes]:
+ """Names a binding target stores to: ``rec.x`` as a whole, else its identifiers."""
+ if target.type in self.key_hops:
+ return {target.text or b""}
+ if target.child_by_field_name("function") is not None:
+ # ``groups.setdefault(k, []).append(x)`` grows what the call returns, not ``k``.
+ return set()
+ return {n.text for n in self._walk(target) if n.type == "identifier" and n.text}
+
+ def _reads(self, node: Node, names: set[bytes]) -> bool:
+ """*node* reads one of *names*: a bare name, or a whole ``rec.x``."""
+ return bool(names) and any(
+ n.text in names
+ for n in self._walk(node)
+ if n.type == "identifier" or n.type in self.key_hops
+ )
+
+ def _result_names(self, call: Node, body: Node, hops: int = 0) -> set[bytes]:
+ """Names the body binds from *call*'s result, then *hops* steps through
+ locals that read them (``cursor = page.next`` carries a page to the next turn)."""
+ bindings = [
+ (target, value)
+ for node in self._walk(body, prune=self.scope_kinds)
+ for target, value in (self._bound_by(node),)
+ if target is not None and value is not None
+ ]
+ names = {
+ t for target, value in bindings if self._within(value, call) for t in self._texts(target)
+ }
+ for _ in range(hops):
+ names |= {
+ t
+ for target, value in bindings
+ if self._reads(value, names)
+ for t in self._texts(target)
+ }
+ return names
+
+ def _appended_to(self, call: Node, body: Node, results: set[bytes]) -> set[bytes]:
+ """Sequences the body appends *call* or its results to."""
+ names: set[bytes] = set()
+ for node in self._walk(body, prune=self.scope_kinds):
+ if self.callee_method_name(node) not in self.sequence_appends:
+ continue
+ fn = node.child_by_field_name("function")
+ receiver = fn.child_by_field_name("object") if fn is not None else None
+ args = node.child_by_field_name("arguments")
+ if receiver is None or args is None:
+ continue
+ if self._within(args, call) or self._reads(args, results):
+ names |= self._texts(receiver)
+ return names
+
+ def _stops_at_hit(
+ self, call: Node, loop: Node, body: Node, results: set[bytes], loop_kinds: frozenset[str]
+ ) -> bool:
+ """An exit that ends *loop* returns the call's result, or the branch it
+ sits in directly tests it; a test further out guards more than the exit."""
+ for exit_node in self._walk(body, prune=self.scope_kinds):
+ if exit_node.type not in self._STOPPING_EXITS:
+ continue
+ if exit_node.type == "break_statement" and not self._breaks(
+ exit_node, loop, loop_kinds
+ ):
+ continue
+ if self._within(exit_node, call) or self._reads(exit_node, results):
+ return True
+ condition = self._exit_condition(exit_node, body)
+ if condition is not None and (
+ self._within(condition, call) or self._reads(condition, results)
+ ):
+ return True
+ return False
+
+ def _exit_condition(self, exit_node: Node, body: Node) -> Node | None:
+ """The condition of the nearest branch around *exit_node* inside *body*."""
+ cur = exit_node.parent
+ while cur is not None and cur != body:
+ # Python's ``elif`` tests its own condition, not the ``if``'s.
+ if cur.type in self.branch_kinds or cur.type == "elif_clause":
+ return cur.child_by_field_name("condition")
+ cur = cur.parent
return None
+ def _breaks(self, exit_node: Node, loop: Node, loop_kinds: frozenset[str]) -> bool:
+ """A ``break`` that leaves *loop* itself, not a nested loop or ``switch``."""
+ cur = exit_node.parent
+ while cur is not None and cur != loop:
+ if cur.type in loop_kinds or cur.type in self.switch_kinds:
+ return False
+ cur = cur.parent
+ return cur is not None
+
def batch_form(self, sink: Node, loop: Node, probe: SinkProbe) -> BatchForm | None:
"""The bulk counterpart of *sink* when its only varying key is the loop's element."""
return None
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/csharp.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/csharp.py
index 60e043f2b..d838ccc8d 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/csharp.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/csharp.py
@@ -29,13 +29,18 @@ query calls in loops.
from __future__ import annotations
+import re
from typing import TYPE_CHECKING
from .base import BasePerfDialect
+from .iterables import IterableSources, upper_bound
if TYPE_CHECKING:
from tree_sitter import Node
+ from ..loop_facts import SinkProbe
+ from .iterables import Source
+
# The EF ``*Async`` LINQ family — EF-only, so UNAMBIGUOUS db (no gate).
EF_ASYNC_METHODS: frozenset[str] = frozenset(
{
@@ -172,7 +177,21 @@ CSHARP_RESOURCE_CTORS: frozenset[str] = frozenset(
)
-class CSharpPerfDialect(BasePerfDialect):
+# Function-like scopes: an ``await`` outside the one holding the read says
+# nothing about it, and an ``await`` inside a nested lambda may never run.
+_CS_FN_KINDS: frozenset[str] = frozenset(
+ {
+ "method_declaration",
+ "local_function_statement",
+ "lambda_expression",
+ "anonymous_method_expression",
+ "constructor_declaration",
+ "accessor_declaration",
+ "operator_declaration",
+ }
+)
+
+class CSharpPerfDialect(IterableSources):
language = "csharp"
markers = frozenset(
{
@@ -358,6 +377,133 @@ class CSharpPerfDialect(BasePerfDialect):
return True
return False
+ # -- what a loop walks (perf/dialects/iterables.py) ---------------------------
+
+ function_kinds = _CS_FN_KINDS
+ change_kinds = frozenset(
+ {"assignment_expression", "postfix_unary_expression", "prefix_unary_expression"}
+ )
+ grow_methods = frozenset({"Add", "AddRange", "Insert", "Push", "Enqueue", "TryAdd"})
+ conditional_kinds = frozenset(
+ {
+ "if_statement", "switch_statement", "switch_section", "switch_expression",
+ "for_statement", "foreach_statement", "while_statement", "do_statement",
+ "try_statement", "catch_clause", "conditional_expression",
+ }
+ )
+
+ literal_kinds = frozenset(
+ {
+ "implicit_array_creation_expression",
+ "array_creation_expression",
+ "object_creation_expression",
+ "collection_expression",
+ }
+ )
+ int_kinds = frozenset({"integer_literal"})
+ wrapper_kinds = frozenset({"parenthesized_expression", "await_expression"})
+ pass_through = frozenset(
+ {
+ "Where", "Select", "SelectMany", "OrderBy", "OrderByDescending", "ThenBy",
+ "ThenByDescending", "Distinct", "Cast", "OfType", "Reverse", "AsEnumerable",
+ "AsQueryable", "AsNoTracking", "AsSplitQuery", "Include", "ThenInclude",
+ "ConfigureAwait", "GetEnumerator", "Keys", "Values",
+ }
+ )
+ caps = frozenset({"Take"})
+ size_names = frozenset({"Count", "Length", "LongCount"})
+ # ``while (reader.Read())`` (ADO.NET), ``while (e.MoveNext())``.
+ cursor_advances = frozenset({"Read", "ReadAsync", "MoveNext", "MoveNextAsync"})
+ listings = frozenset(
+ {
+ "GetFiles", "EnumerateFiles", "GetDirectories", "EnumerateDirectories",
+ "GetFileSystemEntries", "EnumerateFileSystemEntries",
+ }
+ )
+ recursive_marks = (b"AllDirectories",)
+ streams = frozenset({"ReadLines"})
+ query_caps = frozenset({"Take"})
+
+ def loop_header(self, loop: Node) -> tuple[str, Node | None]:
+ if loop.type == "foreach_statement":
+ return "each", loop.child_by_field_name("right")
+ cond = loop.child_by_field_name("condition")
+ if loop.type == "for_statement":
+ return "counted", upper_bound(cond)
+ if loop.type in ("while_statement", "do_statement") and cond is not None:
+ return "cursor", cond
+ return "each", None
+
+ def is_literal(self, node: Node) -> bool:
+ """``new[] { a, b }`` / ``new List<T> { a }`` / ``[a, b]``; not ``[..xs]``
+ and not ``new List<T>(xs)``."""
+ if node.type not in self.literal_kinds:
+ return False
+ if node.type == "collection_expression":
+ return bool(node.named_children) and not any(
+ child.type == "spread_element"
+ for element in node.named_children
+ for child in element.named_children
+ )
+ if node.type == "implicit_array_creation_expression":
+ return True
+ return any(child.type == "initializer_expression" for child in node.named_children)
+
+ def call_parts(self, node: Node) -> tuple[str, Node | None, list[Node]] | None:
+ if node.type != "invocation_expression":
+ return None
+ fn = node.child_by_field_name("function")
+ args = node.child_by_field_name("arguments")
+ arg_nodes = [
+ arg.named_children[-1]
+ for arg in (args.named_children if args is not None else ())
+ if arg.type == "argument" and arg.named_children
+ ]
+ if fn is not None and fn.type == "member_access_expression":
+ return _plain_name(fn.child_by_field_name("name")), _member_object(fn), arg_nodes
+ return _plain_name(fn), None, arg_nodes
+
+ def member_parts(self, node: Node) -> tuple[str, Node] | None:
+ if node.type != "member_access_expression":
+ return None
+ obj = _member_object(node)
+ return (_member_name(node), obj) if obj is not None else None
+
+ def special_call(
+ self,
+ call: Node,
+ parts: tuple[str, Node | None, list[Node]],
+ loop: Node,
+ probe: SinkProbe,
+ ) -> Source | None:
+ method, receiver, _args = parts
+ if method == "GetValues" and _text(receiver) == "Enum":
+ return "constant", "bounded"
+ # Sync LINQ reads the database only on an EF query; on a list it walks the list.
+ if method in SYNC_LINQ_METHODS and receiver is not None and not self.query_chain(call):
+ return self._iterable(receiver, loop, probe, hop=True)
+ return None
+
+ def query_chain(self, node: Node) -> bool:
+ """``_context.Orders.Where(...)``: a LINQ chain on a DbContext's set."""
+ cur, hops = node, 0
+ while True:
+ if cur.type == "invocation_expression":
+ cur = cur.child_by_field_name("function")
+ elif cur.type == "member_access_expression":
+ cur, hops = _member_object(cur), hops + 1
+ else:
+ break
+ if cur is None:
+ return False
+ return hops >= 1 and cur.type == "identifier" and bool(_CS_DB_ROOT.match(_text(cur)))
+
+ def bindings_of(self, node: Node) -> list[tuple[bytes, Node]]:
+ return _cs_bindings(node)
+
+ def param_names(self, loop: Node) -> set[bytes]:
+ return self.enclosing_params(loop, frozenset({"parameter"}))
+
def loop_call_marker(
self, root: str, method: str, node: Node, list_names: frozenset[str]
) -> str | None:
@@ -383,19 +529,6 @@ class CSharpPerfDialect(BasePerfDialect):
return None
-# Function-like scopes: an ``await`` outside the one holding the read says
-# nothing about it, and an ``await`` inside a nested lambda may never run.
-_CS_FN_KINDS: frozenset[str] = frozenset(
- {
- "method_declaration",
- "local_function_statement",
- "lambda_expression",
- "anonymous_method_expression",
- "constructor_declaration",
- "accessor_declaration",
- "operator_declaration",
- }
-)
# Statements that run unconditionally once reached (no nested branch).
_CS_PLAIN_STMTS: frozenset[str] = frozenset({"expression_statement", "local_declaration_statement"})
@@ -414,6 +547,32 @@ def _member_object(node: Node | None) -> Node | None:
return node.child_by_field_name("expression") if node is not None else None
+def _plain_name(node: Node | None) -> str:
+ """``Select`` of ``Select`` and of ``Select<T>``."""
+ if node is not None and node.type == "generic_name":
+ node = next((c for c in node.named_children if c.type == "identifier"), None)
+ return _text(node) if node is not None and node.type == "identifier" else ""
+
+
+# A DbContext held in a field or local (``_context``, ``_db``, ``dbContext``).
+_CS_DB_ROOT = re.compile(r"^_?(?:\w*db|\w*dbcontext|context|ctx)$", re.IGNORECASE)
+
+
+def _cs_bindings(node: Node) -> list[tuple[bytes, Node]]:
+ """What *node* binds: ``var x = v`` or ``x = v``."""
+ if node.type == "variable_declarator":
+ target = node.child_by_field_name("name")
+ value = node.named_children[-1] if node.named_children else None
+ if value is not None and value == target:
+ value = None
+ elif node.type == "assignment_expression":
+ target, value = node.child_by_field_name("left"), node.child_by_field_name("right")
+ else:
+ return []
+ ok = target is not None and target.type == "identifier" and target.text and value is not None
+ return [(target.text, value)] if ok else []
+
+
def _strip_configure_await(node: Node | None) -> Node | None:
# ``await x.ConfigureAwait(false)`` awaits ``x``.
if node is not None and node.type == "invocation_expression":
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/go.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/go.py
index b00481b8b..8c9095ae6 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/go.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/go.py
@@ -21,7 +21,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-from .base import BasePerfDialect
+from .iterables import IterableSources, text_of, upper_bound
from .python import HTTP_VERBS
if TYPE_CHECKING:
@@ -81,6 +81,8 @@ GO_IOUTIL_METHODS: frozenset[str] = frozenset({"ReadFile", "WriteFile", "ReadAll
GO_REGEX_COMPILE: frozenset[str] = frozenset(
{"MustCompile", "Compile", "MustCompilePOSIX", "CompilePOSIX"}
)
+# GORM reads that fill the slice passed as ``&x``; ``Create(&x)`` writes it.
+GO_READ_INTO: frozenset[bytes] = frozenset({b"Find", b"Scan", b"Pluck", b"FindInBatches"})
_GO_STRING_KINDS: frozenset[str] = frozenset({"interpreted_string_literal", "raw_string_literal"})
_GO_LOOP_KINDS: frozenset[str] = frozenset({"for_statement"})
@@ -113,7 +115,54 @@ def _single_identifier(node: Node | None) -> bytes | None:
return targets[0].text
-class GoPerfDialect(BasePerfDialect):
+_GO_FN_KINDS: frozenset[str] = frozenset({"function_declaration", "method_declaration", "func_literal"})
+
+
+def _go_bindings(node: Node) -> list[tuple[bytes, Node]]:
+ """What *node* binds: ``x := v`` / ``x = v`` / ``var x = v`` (one call
+ answering several names counts for each), or a read filling ``&x`` (GORM
+ ``db.Find(&users)``)."""
+ if node.type in ("short_var_declaration", "assignment_statement"):
+ left = node.child_by_field_name("left")
+ targets = list(left.named_children) if left is not None else []
+ return _paired(targets, node.child_by_field_name("right"))
+ if node.type == "var_spec":
+ return _paired(list(node.children_by_field_name("name")), node.child_by_field_name("value"))
+ if node.type == "call_expression" and _callee_name(node) in GO_READ_INTO:
+ args = node.child_by_field_name("arguments")
+ filled = [_address_of(arg) for arg in (args.named_children if args is not None else ())]
+ return [(name, node) for name in filled if name]
+ return []
+
+
+def _callee_name(call: Node) -> bytes | None:
+ fn = call.child_by_field_name("function")
+ field = fn.child_by_field_name("field") if fn is not None else None
+ return field.text if field is not None else None
+
+
+def _address_of(arg: Node) -> bytes | None:
+ """``x`` of ``&x``."""
+ if arg.type != "unary_expression" or not (arg.text or b"").startswith(b"&"):
+ return None
+ operand = arg.child_by_field_name("operand")
+ return operand.text if operand is not None and operand.type == "identifier" else None
+
+
+def _paired(targets: list[Node], right: Node | None) -> list[tuple[bytes, Node]]:
+ if right is None:
+ return []
+ values = right.named_children if right.type == "expression_list" else [right]
+ if len(values) != len(targets) and len(values) != 1:
+ return []
+ return [
+ (target.text, values[index] if len(values) == len(targets) else values[0])
+ for index, target in enumerate(targets)
+ if target.type == "identifier" and target.text
+ ]
+
+
+class GoPerfDialect(IterableSources):
language = "go"
markers = frozenset(
{
@@ -279,6 +328,85 @@ class GoPerfDialect(BasePerfDialect):
return False
return self.callee_method_name(right) in GO_CHUNK_CALLS
+ # -- what a loop walks (perf/dialects/iterables.py) ---------------------------
+
+ function_kinds = _GO_FN_KINDS
+ change_kinds = frozenset({"assignment_statement", "inc_statement", "dec_statement"})
+ conditional_kinds = frozenset(
+ {
+ "if_statement", "expression_switch_statement", "type_switch_statement",
+ "select_statement", "expression_case", "type_case", "default_case",
+ "communication_case", "for_statement",
+ }
+ )
+
+ literal_kinds = frozenset({"composite_literal"})
+ int_kinds = frozenset({"int_literal"})
+ wrapper_kinds = frozenset({"parenthesized_expression"})
+ # ``maps.Keys(m)`` / ``slices.Sorted(maps.Keys(m))`` walk their argument.
+ arg_walkers = frozenset({"Keys", "Values", "Sorted", "SortedFunc", "Collect", "Clone"})
+ size_names = frozenset({"len"})
+ # ``for rows.Next()`` (database/sql), ``for scanner.Scan()`` (bufio).
+ cursor_advances = frozenset({"Next", "Scan"})
+ listings = frozenset({"ReadDir", "Glob"})
+ streams = frozenset({"NewScanner", "NewReader"})
+ query_caps = frozenset({"Limit"})
+
+ def loop_header(self, loop: Node) -> tuple[str, Node | None]:
+ if loop.type != "for_statement":
+ return "each", None
+ body = loop.child_by_field_name("body")
+ for child in loop.named_children:
+ if child.type == "range_clause":
+ return "each", child.child_by_field_name("right")
+ if child.type == "for_clause":
+ return "counted", upper_bound(child.child_by_field_name("condition"))
+ if child.type == "call_expression" and child != body:
+ return "cursor", child
+ return "each", None
+
+ def is_literal(self, node: Node) -> bool:
+ """``[]string{"a", "b"}``; an empty ``[]T{}`` is filled later."""
+ body = node.child_by_field_name("body") if node.type == "composite_literal" else None
+ return body is not None and any(c.type != "comment" for c in body.named_children)
+
+ def call_parts(self, node: Node) -> tuple[str, Node | None, list[Node]] | None:
+ if node.type != "call_expression":
+ return None
+ fn = node.child_by_field_name("function")
+ args = node.child_by_field_name("arguments")
+ arg_nodes = [c for c in args.named_children if c.type != "comment"] if args else []
+ if fn is not None and fn.type == "selector_expression":
+ field = fn.child_by_field_name("field")
+ return text_of(field), fn.child_by_field_name("operand"), arg_nodes
+ return text_of(fn), None, arg_nodes
+
+ def fixed_slice(self, node: Node) -> Node | None:
+ """``xs[:5]`` / ``xs[0:5]``."""
+ if node.type != "slice_expression":
+ return None
+ start, end = node.child_by_field_name("start"), node.child_by_field_name("end")
+ if end is None or end.type != "int_literal":
+ return None
+ if start is not None and start.text != b"0":
+ return None
+ return node.child_by_field_name("operand")
+
+ def bindings_of(self, node: Node) -> list[tuple[bytes, Node]]:
+ return _go_bindings(node)
+
+ def param_names(self, loop: Node) -> set[bytes]:
+ scope = self._scope(loop)
+ params = scope.child_by_field_name("parameters") if scope is not None else None
+ if params is None:
+ return set()
+ return {
+ name.text
+ for decl in params.named_children
+ for name in decl.children_by_field_name("name")
+ if name.text
+ }
+
@staticmethod
def _has_static_pattern_arg(node: Node) -> bool:
"""True if the call's first argument is a compile-time string literal.
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/iterables.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/iterables.py
new file mode 100644
index 000000000..76f7ef396
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/iterables.py
@@ -0,0 +1,449 @@
+"""What a loop walks, for dialects that describe their syntax through hooks.
+
+Go, Java, C# and Ruby share one reading of a loop header: a literal is a
+constant, a fixed-width cap bounds the trip count of whatever it caps, a read
+that leaves the process returns rows that grow unless it is capped or keyed
+on one id, a parameter is whatever the caller passes, and a cursor walks what
+its receiver was bound to. Each dialect names its own node kinds and method
+names (and its binding statements); anything this reading does not settle
+stays ``unknown``, which no reader takes as bounded.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, ClassVar
+
+from .. import loop_lexicon as lex
+from ..loop_facts import LoopSource
+from .base import BasePerfDialect
+
+if TYPE_CHECKING:
+ from tree_sitter import Node
+
+ from ..loop_facts import LoopIterates, LoopMagnitude, SinkProbe
+
+ Source = tuple[LoopIterates, LoopMagnitude | None]
+
+_UNKNOWN: Source = ("unknown", None)
+_BLOCKS = frozenset({"block", "do_block"})
+_SPLATS = frozenset({"splat_argument", "hash_splat_argument", "variadic_argument"})
+# What a capping call's receiver may be when the cap belongs to a read.
+_READS = frozenset({"unknown", "query", "caller_key", "rare_subset"})
+
+
+def text_of(node: Node | None) -> str:
+ return (node.text or b"").decode("utf-8", "replace") if node is not None else ""
+
+
+def upper_bound(cond: Node | None) -> Node | None:
+ """``n`` of a counted loop's ``i < n`` / ``i <= n``; ``None`` for any other condition."""
+ if cond is None or cond.type != "binary_expression":
+ return None
+ if not any(c.type in ("<", "<=") for c in cond.children):
+ return None
+ return cond.child_by_field_name("right")
+
+
+@dataclass(slots=True)
+class ScopeFacts:
+ """One function's bindings and mutations, read in one pass over its body.
+
+ ``bindings``: name -> ``(statement, value)`` in source order; ``changes``:
+ name -> start offsets of statements that grow or step it.
+ """
+
+ bindings: dict[bytes, list[tuple[Node, Node]]] = field(default_factory=dict)
+ changes: dict[bytes, list[int]] = field(default_factory=dict)
+
+
+class IterableSources(BasePerfDialect):
+ """``loop_source`` read off a dialect's loop header.
+
+ A dialect supplies :meth:`loop_header`, :meth:`call_parts`,
+ :meth:`reaching_value` and :meth:`param_names`, plus the vocabularies below.
+ """
+
+ # Function scopes a binding or parameter belongs to.
+ function_kinds: frozenset[str] = frozenset()
+ # Inline collection literals, integer literals and name nodes.
+ literal_kinds: frozenset[str] = frozenset()
+ int_kinds: frozenset[str] = frozenset()
+ name_kinds: frozenset[str] = frozenset({"identifier"})
+ # Wrappers read through (``(x)``, ``await x``).
+ wrapper_kinds: frozenset[str] = frozenset()
+ # ``recv.m()`` walks ``recv``; ``m(x)`` walks ``x``; ``m(n)`` keeps at most n.
+ pass_through: frozenset[str] = frozenset()
+ arg_walkers: frozenset[str] = frozenset()
+ caps: frozenset[str] = frozenset()
+ # ``len(xs)`` / ``xs.size()``: the size of what the counted loop walks.
+ size_names: frozenset[str] = frozenset()
+ # ``rows.Next()`` / ``rs.next()``: a cursor that walks its receiver.
+ cursor_advances: frozenset[str] = frozenset()
+ listings: frozenset[str] = frozenset()
+ streams: frozenset[str] = frozenset()
+ # Method names that cap a read in the query (``Limit(10)``, ``take(5)``).
+ query_caps: frozenset[str] = frozenset()
+ # Text that makes a listing recursive (``**``, ``AllDirectories``).
+ recursive_marks: ClassVar[tuple[bytes, ...]] = (b"**",)
+ recursive_listings: frozenset[str] = frozenset()
+ # Property reads a listing's path may name the repository root through.
+ member_kinds: frozenset[str] = frozenset()
+ # Statements that step or grow the name on their left (``n++``, ``m[k] = v``),
+ # and methods that grow their receiver (``xs.Add(x)``).
+ change_kinds: frozenset[str] = frozenset()
+ grow_methods: frozenset[str] = frozenset()
+ # Nodes whose body may run zero times or one of several ways; a binding
+ # under one does not surely reach code after it (its condition does).
+ conditional_kinds: frozenset[str] = frozenset()
+ # ``None``: a dialect whose own syntax rules size a read (Python, TS).
+ read_grows: LoopMagnitude | None = "grows_with_data"
+
+ # The last scope read, held so its tree (and so the node id in the key) stays live.
+ _facts_cache: tuple[Node, tuple[int, int, int], ScopeFacts] | None = None
+
+ # -- hooks -----------------------------------------------------------------
+
+ def loop_header(self, loop: Node) -> tuple[str, Node | None]:
+ """``("each", iterable)``, ``("counted", stop)`` or ``("cursor", call)``."""
+ return "each", None
+
+ def call_parts(self, node: Node) -> tuple[str, Node | None, list[Node]] | None:
+ """``(method, receiver, args)`` when *node* is a call, else ``None``."""
+ return None
+
+ def is_literal(self, node: Node) -> bool:
+ """A collection the source spells out entry by entry. An empty one is
+ filled later, and a splat (``[*xs]``) adds as many entries as ``xs`` holds."""
+ entries = [c for c in node.named_children if c.type != "comment"]
+ return (
+ node.type in self.literal_kinds
+ and bool(entries)
+ and not any(child.type in _SPLATS for child in entries)
+ )
+
+ def member_parts(self, node: Node) -> tuple[str, Node] | None:
+ """``(name, object)`` when *node* reads a property (``xs.Count``), else ``None``."""
+ return None
+
+ def bindings_of(self, node: Node) -> list[tuple[bytes, Node]]:
+ """``(name, value)`` for each bare name the statement *node* binds."""
+ return []
+
+ def mutated_name(self, node: Node) -> bytes | None:
+ """The name *node* grows or steps (``xs.Add(x)``, ``m[k] = v``, ``n++``)."""
+ if node.type in self.change_kinds and node.named_children:
+ return (node.named_children[0].text or b"").split(b"[", 1)[0]
+ parts = self.call_parts(node) if self.grow_methods else None
+ if parts is not None and parts[0] in self.grow_methods and parts[1] is not None:
+ return parts[1].text
+ return None
+
+ def reads_config_table(self, call: Node, nodes: list[Node]) -> bool:
+ """The read names only registry or configuration tables."""
+ return False
+
+ def is_cap(self, node: Node, loop: Node) -> bool:
+ """*node* caps a read at a fixed count (``Limit(10)``, ``setMaxResults(5)``)."""
+ parts = self.call_parts(node)
+ return parts is not None and parts[0] in self.query_caps and bool(parts[2])
+
+ def param_names(self, loop: Node) -> set[bytes]:
+ """The parameters of the function *loop* sits in."""
+ return set()
+
+ def special_call(
+ self,
+ call: Node,
+ parts: tuple[str, Node | None, list[Node]],
+ loop: Node,
+ probe: SinkProbe,
+ ) -> Source | None:
+ """A dialect's own reading of a call (``List.of(a, b)``, ``Role.values()``)."""
+ return None
+
+ def query_chain(self, node: Node) -> bool:
+ """A read the sink probe does not see (an ActiveRecord relation)."""
+ return False
+
+ def fixed_slice(self, node: Node) -> Node | None:
+ """The operand of a fixed-width slice (``xs[:5]``), else ``None``."""
+ return None
+
+ # -- shared reading ----------------------------------------------------------
+
+ def loop_source(self, loop: Node, probe: SinkProbe) -> LoopSource | None:
+ kind, node = self.loop_header(loop)
+ if node is None:
+ return None
+ if kind == "counted":
+ iterates, magnitude = self._counted(node, loop, probe)
+ elif kind == "cursor":
+ iterates, magnitude = self._cursor(node, loop, probe)
+ else:
+ iterates, magnitude = self._iterable(node, loop, probe, hop=True)
+ if iterates == "unknown" and magnitude is None:
+ return None
+ return LoopSource(iterates, magnitude or "unknown")
+
+ def _unwrap(self, node: Node) -> Node:
+ while node.type in self.wrapper_kinds:
+ inner = next((c for c in node.named_children if c.type != "comment"), None)
+ if inner is None:
+ break
+ node = inner
+ return node
+
+ def _iterable(self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ if expr is None:
+ return _UNKNOWN
+ node = self._unwrap(expr)
+ if self.is_literal(node):
+ return "constant", "bounded"
+ if node.type in self.int_kinds:
+ return "constant", "bounded" # Go ``range 10``
+ if node.type in self.name_kinds:
+ return self._name(node, loop, probe, hop=hop)
+ sliced = self.fixed_slice(node)
+ if sliced is not None:
+ # A fixed-width slice bounds the trip count, not what is walked.
+ return self._iterable(sliced, loop, probe, hop=hop)[0], "bounded"
+ parts = self.call_parts(node)
+ if parts is not None:
+ return self._call(node, parts, loop, probe, hop=hop)
+ member = self.member_parts(node)
+ if member is not None and member[0] in self.pass_through:
+ return self._iterable(member[1], loop, probe, hop=hop)
+ if member is not None and self.query_chain(node):
+ return self._query(node, loop)
+ if lex.is_config_name(text_of(node)):
+ return "config_table", None
+ return _UNKNOWN
+
+ def _name(self, name: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ value = self.reaching_value(loop, name.text or b"") if hop else None
+ if value is not None:
+ found = self._iterable(value, loop, probe, hop=False)
+ if found[0] == "constant" and self._changed_before(value, loop, name.text or b""):
+ found = _UNKNOWN
+ if found != _UNKNOWN:
+ return found
+ if (name.text or b"") in self.param_names(loop):
+ return "parameter", None
+ if lex.is_config_name(text_of(name)):
+ # A registry grows with what users register: the name alone bounds nothing.
+ return "config_table", None
+ return _UNKNOWN
+
+ def _call(
+ self,
+ call: Node,
+ parts: tuple[str, Node | None, list[Node]],
+ loop: Node,
+ probe: SinkProbe,
+ *,
+ hop: bool,
+ ) -> Source:
+ method, receiver, args = parts
+ found = self.special_call(call, parts, loop, probe) or self._read_or_cap(
+ call, parts, loop, probe, hop=hop
+ )
+ if found is not None:
+ return found
+ if method in self.pass_through and receiver is not None:
+ return self._iterable(receiver, loop, probe, hop=hop)
+ if method in self.arg_walkers and args:
+ return self._iterable(args[0], loop, probe, hop=hop)
+ if method in self.listings:
+ return "listing", "grows_with_data" if self._walks_repo(call, method, loop) else None
+ if method in self.streams:
+ return "stream", None
+ if lex.is_config_name(method, callee=True):
+ return "config_table", None
+ return _UNKNOWN
+
+ def _read_or_cap(
+ self,
+ call: Node,
+ parts: tuple[str, Node | None, list[Node]],
+ loop: Node,
+ probe: SinkProbe,
+ *,
+ hop: bool,
+ ) -> Source | None:
+ """A read that leaves the process, or a cap on what the loop walks.
+
+ A cap on what is already in hand bounds the walk (``ids.first(5)``); a
+ cap inside a read is the read's (``User.limit(10)``).
+ """
+ method, receiver, args = parts
+ capped = method in self.caps and receiver is not None and self._int_arg(args)
+ walked = self._iterable(receiver, loop, probe, hop=hop)[0] if capped else "unknown"
+ if capped and walked not in _READS:
+ return walked, "bounded"
+ if probe(call) in ("db", "network") or self.query_chain(call):
+ return self._query(call, loop)
+ if capped and method in self.query_caps:
+ return walked, None # likely a read's own cap, which proves nothing alone
+ return (walked, "bounded") if capped else None
+
+ def _int_arg(self, args: list[Node]) -> bool:
+ return len(args) == 1 and self._unwrap(args[0]).type in self.int_kinds
+
+ def _counted(self, stop: Node, loop: Node, probe: SinkProbe) -> Source:
+ """``i < 3`` is a constant; ``i < len(rows)`` walks ``rows``; ``i < n`` a parameter."""
+ node = self._unwrap(stop)
+ if node.type in self.int_kinds:
+ return "constant", "bounded"
+ if node.type in self.name_kinds:
+ if (node.text or b"") in self.param_names(loop):
+ return "parameter", None
+ value = self.reaching_value(loop, node.text or b"")
+ if value is None or self._changed_before(value, loop, node.text or b""):
+ return _UNKNOWN
+ return self._counted(value, loop, probe)
+ sized = self._sized(node)
+ return self._iterable(sized, loop, probe, hop=True) if sized is not None else _UNKNOWN
+
+ def _sized(self, node: Node) -> Node | None:
+ """``xs`` of ``len(xs)`` / ``xs.size()`` / ``xs.Count`` / ``xs.length``."""
+ parts = self.call_parts(node)
+ if parts is not None:
+ method, receiver, args = parts
+ if method not in self.size_names:
+ return None
+ if receiver is not None and not args:
+ return receiver
+ return args[0] if receiver is None and len(args) == 1 else None
+ member = self.member_parts(node)
+ return member[1] if member is not None and member[0] in self.size_names else None
+
+ def _cursor(self, call: Node, loop: Node, probe: SinkProbe) -> Source:
+ """``for rows.Next()`` / ``while (rs.next())``: the read the receiver holds."""
+ parts = self.call_parts(self._unwrap(call))
+ if parts is None or parts[0] not in self.cursor_advances or parts[1] is None:
+ return _UNKNOWN
+ return self._iterable(parts[1], loop, probe, hop=True)
+
+ def _query(self, call: Node, loop: Node) -> Source:
+ """Rows of one key, a capped read, a rare state, or rows that grow; the
+ names the call reads may be bound to their statement before the loop."""
+ # A Ruby ``find_each do ... end`` reads before its block runs.
+ block = call.child_by_field_name("block")
+ head = (call.text or b"")[: block.start_byte - call.start_byte] if block else call.text
+ names = [n.text for n in self._walk(call, prune=_BLOCKS) if n.type in self.name_kinds]
+ values = [v for name in dict.fromkeys(names) if (v := self.reaching_value(loop, name or b""))]
+ nodes = [call, *values]
+ if self.reads_config_table(call, nodes):
+ return "config_table", None
+ text = b" ".join([head or b"", *(value.text or b"" for value in values)])
+ if lex.is_key_lookup(text):
+ return "caller_key", "bounded"
+ # The children of one parent (``user_id = ?``) are as many as that parent
+ # has, and a capped read is as large as its cap: neither settles a size.
+ capped = lex.has_sql_limit(text) or any(
+ self.is_cap(sub, loop) for node in nodes for sub in self._walk(node, prune=_BLOCKS)
+ )
+ if capped or lex.is_parent_key_filter(text):
+ return "query", None
+ return ("rare_subset" if lex.is_rare_filter(text) else "query"), self.read_grows
+
+ def _walks_repo(self, call: Node, method: str, loop: Node) -> bool:
+ """A recursive listing whose path names a repository root; any other path
+ proves nothing about its size."""
+ text = call.text or b""
+ if method not in self.recursive_listings and not any(
+ mark in text for mark in self.recursive_marks
+ ):
+ return False
+ kinds = self.name_kinds | self.member_kinds
+ names = [n for n in self._walk(call) if n.type in kinds]
+ for node in [n for n in names if n.type in self.name_kinds]:
+ value = self.reaching_value(loop, node.text or b"")
+ if value is not None:
+ names.extend(n for n in self._walk(value) if n.type in kinds)
+ return any(lex.names_repo_root(text_of(n)) for n in names)
+
+ # -- helpers the dialects share ---------------------------------------------
+
+ def _scope(self, loop: Node) -> Node | None:
+ scope = loop.parent
+ while scope is not None and scope.type not in self.function_kinds:
+ scope = scope.parent
+ return scope
+
+ def scope_facts(self, anchor: Node) -> ScopeFacts | None:
+ """The bindings and mutations of the function around *anchor*, read once
+ per function: every loop and call in it asks the same map."""
+ scope = self._scope(anchor)
+ if scope is None:
+ return None
+ key = (scope.id, scope.start_byte, scope.end_byte)
+ cached = self._facts_cache
+ if cached is not None and cached[1] == key:
+ return cached[2]
+ facts = ScopeFacts()
+ for node in self._walk(scope, prune=self.function_kinds):
+ for name, value in self.bindings_of(node):
+ facts.bindings.setdefault(name, []).append((node, value))
+ changed = self.mutated_name(node)
+ if changed:
+ facts.changes.setdefault(changed, []).append(node.start_byte)
+ self._facts_cache = (scope, key, facts)
+ return facts
+
+ def bindings_before(self, anchor: Node, name: bytes) -> list[tuple[Node, Node]]:
+ """Every ``(statement, value)`` binding *name* that ends before *anchor*."""
+ facts = self.scope_facts(anchor)
+ found = facts.bindings.get(name, ()) if facts is not None else ()
+ return [(stmt, value) for stmt, value in found if stmt.end_byte <= anchor.start_byte]
+
+ def reaching_value(self, loop: Node, name: bytes) -> Node | None:
+ """The value *name* surely holds at *loop*: its one binding before the
+ loop, made where the loop also runs. A second binding, or one under a
+ branch or loop the target loop is not in, leaves it unknown."""
+ before = self.bindings_before(loop, name)
+ if len(before) != 1 or not self._surely_reaches(before[0][0], loop):
+ return None
+ return before[0][1]
+
+ def _surely_reaches(self, stmt: Node, loop: Node) -> bool:
+ """Every node between *stmt* and the first one holding *loop* runs
+ whenever that one does (a branch's condition or initializer does)."""
+ prev, cur = stmt, stmt.parent
+ while cur is not None and not (
+ cur.start_byte <= loop.start_byte and loop.end_byte <= cur.end_byte
+ ):
+ if cur.type in self.conditional_kinds and prev not in (
+ cur.child_by_field_name("condition"),
+ cur.child_by_field_name("initializer"),
+ ):
+ return False
+ prev, cur = cur, cur.parent
+ return cur is not None
+
+ def _changed_before(self, value: Node, loop: Node, name: bytes) -> bool:
+ """*name* is grown or stepped between its binding to *value* and *loop*:
+ a literal it was bound to then says nothing about its size."""
+ facts = self.scope_facts(loop)
+ if facts is None:
+ return True
+ start, end = value.end_byte, loop.start_byte
+ return any(start <= at < end for at in facts.changes.get(name, ()))
+
+ def enclosing_params(self, loop: Node, param_kinds: frozenset[str]) -> set[bytes]:
+ """Names the function around *loop* takes as parameters."""
+ scope = self._scope(loop)
+ params = scope.child_by_field_name("parameters") if scope is not None else None
+ if params is None:
+ return set()
+ names: set[bytes] = set()
+ for node in self._walk(params):
+ if node.type in param_kinds:
+ name = node.child_by_field_name("name")
+ if name is not None and name.text:
+ names.add(name.text)
+ return names
+
+
+__all__ = ["IterableSources", "ScopeFacts", "text_of", "upper_bound"]
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/java.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/java.py
index d4d58c691..cae8a8733 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/java.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/java.py
@@ -26,11 +26,14 @@ from __future__ import annotations
import re
from typing import TYPE_CHECKING
-from .base import BasePerfDialect
+from .iterables import IterableSources, text_of, upper_bound
if TYPE_CHECKING:
from tree_sitter import Node
+ from ..loop_facts import SinkProbe
+ from .iterables import Source
+
# Spring-Data derived query methods: ``findByName`` / ``getAllByStatus`` /
# ``countByOwner`` … The name alone is not enough (``Settings.getByPrefix``,
# ``InetAddress.getByAddress`` and ``RoutingNodes.getByShardId`` are in-memory),
@@ -218,7 +221,32 @@ def _receiver_root(receiver: str) -> str:
return parts[1] if parts[0] == "this" and len(parts) > 1 else parts[0]
-class JavaPerfDialect(BasePerfDialect):
+_JAVA_FN_KINDS: frozenset[str] = frozenset(
+ {"method_declaration", "constructor_declaration", "lambda_expression"}
+)
+# ``List.of(a, b)`` / ``Arrays.asList(a, b)``: as many entries as the call spells out.
+_JAVA_FACTORIES: frozenset[str] = frozenset(
+ {"List", "Set", "Map", "Arrays", "Stream", "EnumSet", "ImmutableList", "ImmutableSet"}
+)
+_JAVA_SCALARS: frozenset[str] = frozenset(
+ {"string_literal", "decimal_integer_literal", "character_literal", "true", "false"}
+)
+# A type name (``Role``), not a constant (``ROLES``) or a local (``roles``).
+_JAVA_TYPE_NAME = re.compile(r"^[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$")
+
+
+def _java_bindings(node: Node) -> list[tuple[bytes, Node]]:
+ """What *node* binds: ``T x = v`` or ``x = v``."""
+ if node.type == "variable_declarator":
+ target, value = node.child_by_field_name("name"), node.child_by_field_name("value")
+ elif node.type == "assignment_expression":
+ target, value = node.child_by_field_name("left"), node.child_by_field_name("right")
+ else:
+ return []
+ ok = target is not None and target.type == "identifier" and target.text and value is not None
+ return [(target.text, value)] if ok else []
+
+class JavaPerfDialect(IterableSources):
language = "java"
lock_acquire_functions = JAVA_LOCK_ACQUIRE_FUNCTIONS
spin_loop_header = JAVA_SPIN_LOOP_HEADER
@@ -364,6 +392,93 @@ class JavaPerfDialect(BasePerfDialect):
return "lock_in_loop"
return None
+ # -- what a loop walks (perf/dialects/iterables.py) ---------------------------
+
+ function_kinds = _JAVA_FN_KINDS
+ change_kinds = frozenset({"assignment_expression", "update_expression"})
+ grow_methods = frozenset({"add", "addAll", "put", "putAll", "push", "offer", "addFirst", "addLast"})
+ conditional_kinds = frozenset(
+ {
+ "if_statement", "switch_expression", "switch_block_statement_group", "switch_rule",
+ "for_statement", "enhanced_for_statement", "while_statement", "do_statement",
+ "try_statement", "catch_clause", "ternary_expression",
+ }
+ )
+
+ literal_kinds = frozenset({"array_initializer"})
+ int_kinds = frozenset({"decimal_integer_literal", "hex_integer_literal"})
+ wrapper_kinds = frozenset({"parenthesized_expression"})
+ pass_through = frozenset(
+ {
+ "stream", "parallelStream", "filter", "map", "flatMap", "sorted", "distinct",
+ "peek", "boxed", "collect", "toList", "entrySet", "keySet", "values", "iterator",
+ "listIterator",
+ }
+ )
+ caps = frozenset({"limit"})
+ size_names = frozenset({"size", "length"})
+ # ``while (rs.next())`` (JDBC), ``while (it.hasNext())``.
+ cursor_advances = frozenset({"next", "hasNext", "hasMoreElements"})
+ listings = frozenset({"listFiles"})
+ recursive_listings = frozenset({"walk", "find"})
+ streams = frozenset({"lines", "readAllLines"})
+ query_caps = frozenset({"setMaxResults"})
+
+ def loop_header(self, loop: Node) -> tuple[str, Node | None]:
+ if loop.type == "enhanced_for_statement":
+ return "each", loop.child_by_field_name("value")
+ cond = loop.child_by_field_name("condition")
+ if loop.type == "for_statement":
+ return "counted", upper_bound(cond)
+ if loop.type in ("while_statement", "do_statement") and cond is not None:
+ return "cursor", cond
+ return "each", None
+
+ def call_parts(self, node: Node) -> tuple[str, Node | None, list[Node]] | None:
+ if node.type != "method_invocation":
+ return None
+ args = node.child_by_field_name("arguments")
+ arg_nodes = [c for c in args.named_children if "comment" not in c.type] if args else []
+ return (
+ text_of(node.child_by_field_name("name")),
+ node.child_by_field_name("object"),
+ arg_nodes,
+ )
+
+ def member_parts(self, node: Node) -> tuple[str, Node] | None:
+ obj = node.child_by_field_name("object") if node.type == "field_access" else None
+ return (text_of(node.child_by_field_name("field")), obj) if obj is not None else None
+
+ def special_call(
+ self,
+ call: Node,
+ parts: tuple[str, Node | None, list[Node]],
+ loop: Node,
+ probe: SinkProbe,
+ ) -> Source | None:
+ method, receiver, args = parts
+ owner = text_of(receiver)
+ if method in ("of", "asList") and owner in _JAVA_FACTORIES:
+ if len(args) != 1:
+ return "constant", "bounded"
+ # ``Arrays.asList(ids)`` walks ``ids``; ``List.of("a")`` is one entry.
+ if args[0].type in _JAVA_SCALARS:
+ return "constant", "bounded"
+ return self._iterable(args[0], loop, probe, hop=True)
+ if method == "values" and not args and _JAVA_TYPE_NAME.match(owner):
+ return "constant", "bounded" # ``Role.values()``: an enum's constants
+ if owner == "Files" and method in ("list", "walk", "find", "newDirectoryStream"):
+ recursive = method in ("walk", "find")
+ grows = recursive and self._walks_repo(call, "walk", loop)
+ return "listing", "grows_with_data" if grows else None
+ return None
+
+ def bindings_of(self, node: Node) -> list[tuple[bytes, Node]]:
+ return _java_bindings(node)
+
+ def param_names(self, loop: Node) -> set[bytes]:
+ return self.enclosing_params(loop, frozenset({"formal_parameter"}))
+
def is_chunked_loop(self, node: Node) -> bool:
"""``for (int i = 0; i < n; i += step)``, or ``for (var c : Lists.partition(xs, n))``."""
if node.type == "for_statement":
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
index f4af3d8b6..27ed8467b 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
@@ -10,16 +10,19 @@ from __future__ import annotations
import builtins
import re
-from collections.abc import Iterator
from typing import TYPE_CHECKING, ClassVar
-from ..loop_facts import BatchForm
+from .. import loop_lexicon as lex
+from ..loop_facts import BatchForm, LoopSource
from .base import BasePerfDialect
+from .iterables import IterableSources
if TYPE_CHECKING:
from tree_sitter import Node
- from ..loop_facts import LoopMagnitude, SinkProbe
+ from ..loop_facts import LoopIterates, LoopMagnitude, SinkProbe
+
+ Source = tuple[LoopIterates, LoopMagnitude | None]
# Network round-trip verbs (shared across languages that name HTTP methods).
HTTP_VERBS: frozenset[str] = frozenset(
@@ -191,6 +194,31 @@ _PY_BUILTINS: frozenset[str] = frozenset(dir(builtins))
_PY_SEM_NAME_RE = re.compile(r"(?i)(sem|semaphore|limiter|limit)$")
# A model class is CapWords; an ALL_CAPS constant (``API_URL``) is not.
_PY_MODEL_NAME_RE = re.compile(r"^[A-Z]\w*[a-z]\w*$")
+# Directory listings; the recursive ones walk a whole tree.
+_PY_LISTINGS: frozenset[str] = frozenset(
+ {
+ "glob", "iglob", "rglob", "iterdir", "walk", "listdir", "scandir", "traverse",
+ "iter_glob", "glob_via", "walk_repo", "_rglob",
+ }
+)
+_PY_RECURSIVE_LISTINGS: frozenset[str] = frozenset(
+ {"rglob", "walk", "traverse", "walk_repo", "_rglob"}
+)
+# Reads that yield lines or chunks as they arrive.
+_PY_STREAMS: frozenset[str] = frozenset(
+ {
+ "readlines", "iter_lines", "iter_content", "iter_bytes", "iter_text",
+ "aiter_lines", "aiter_bytes", "aiter_text", "aiter_raw",
+ }
+)
+# Wrappers and chunkers whose first argument is what the loop walks.
+_PY_PASS_THROUGH: frozenset[str] = frozenset(
+ {
+ "enumerate", "sorted", "reversed", "list", "set", "tuple", "iter", "zip",
+ "batched", "chunked", "ichunked", "chunks", "iter_chunks", "grouper",
+ }
+)
+_PY_CONSTANT_RHS: frozenset[str] = frozenset({"list", "tuple", "set", "dictionary"})
_PY_DEF_KINDS = frozenset({"function_definition", "lambda", "class_definition"})
@@ -257,7 +285,7 @@ def _module_int_constant(name: Node) -> bool:
scope = scope.parent
if scope is None or any(
n.type == "global_statement" and any(c.text == text for c in n.named_children)
- for n in _walk_tree(scope)
+ for n in BasePerfDialect._walk(scope)
):
return False
bound = _bindings(scope, text)
@@ -267,15 +295,26 @@ def _module_int_constant(name: Node) -> bool:
return right is not None and right.type == "integer"
-def _walk_tree(node: Node) -> Iterator[Node]:
- stack = [node]
- while stack:
- current = stack.pop()
- yield current
- stack.extend(current.children)
+def _text(node: Node) -> str:
+ return (node.text or b"").decode("utf-8", "replace")
+
+_PY_SCALARS = frozenset({"string", "concatenated_string", "integer", "float", "true", "false", "none"})
-class PythonPerfDialect(BasePerfDialect):
+
+def _scalar_literal(item: Node) -> bool:
+ """``"a"`` / ``3`` / ``("a", 1)`` / ``"k": v`` by its key: an entry the source
+ spells out. A name or call in the literal (``[ROOT / "a", repo.path]``) may
+ stand for anything, so the table is not shown to be a fixed set."""
+ if item.type == "pair":
+ key = item.child_by_field_name("key")
+ return key is not None and key.type in _PY_SCALARS
+ if item.type == "tuple":
+ return all(child.type in _PY_SCALARS for child in item.named_children)
+ return item.type in _PY_SCALARS
+
+
+class PythonPerfDialect(IterableSources):
language = "python"
lock_acquire_functions = frozenset({"acquire", "__enter__"})
spin_loop_header = re.compile(r"while\s+True\s*:")
@@ -443,7 +482,7 @@ class PythonPerfDialect(BasePerfDialect):
def _bound_locally(self, call: Node, name: bytes) -> bool:
"""*name* is assigned before *call* in its function, or is a parameter of it."""
- if self._reaching_rhs(call, name) is not None:
+ if self.bindings_before(call, name):
return True
scope = call.parent
while scope is not None and scope.type != "function_definition":
@@ -928,10 +967,199 @@ class PythonPerfDialect(BasePerfDialect):
scope_kinds = _PY_SCOPES
sequence_appends = frozenset({"append", "extend", "insert"})
- def loop_magnitude(self, loop: Node, probe: SinkProbe) -> LoopMagnitude | None:
+ # -- what a loop walks: the shared reading (perf/dialects/iterables.py) -------
+ function_kinds = _PY_SCOPES
+ member_kinds = frozenset({"attribute"})
+ recursive_listings = _PY_RECURSIVE_LISTINGS
+ recursive_marks = (b"**", b"recursive=True")
+ conditional_kinds = frozenset(
+ {
+ "if_statement", "elif_clause", "else_clause", "for_statement", "while_statement",
+ "try_statement", "except_clause", "match_statement", "case_clause",
+ "conditional_expression", "boolean_operator",
+ }
+ )
+ # The iterable's own syntax (``_magnitude``) says whether a read grows.
+ read_grows = None
+
+ def bindings_of(self, node: Node) -> list[tuple[bytes, Node]]:
+ if node.type == "assignment":
+ target, value = node.child_by_field_name("left"), node.child_by_field_name("right")
+ elif node.type == "named_expression":
+ target, value = node.child_by_field_name("name"), node.child_by_field_name("value")
+ else:
+ return []
+ ok = target is not None and target.type == "identifier" and target.text and value is not None
+ return [(target.text, value)] if ok else []
+
+ def loop_source(self, loop: Node, probe: SinkProbe) -> LoopSource | None:
if loop.type != "for_statement":
return None
- return self._magnitude(loop.child_by_field_name("right"), loop, probe, hop=True)
+ right = loop.child_by_field_name("right")
+ baseline = self._magnitude(right, loop, probe, hop=True) or "unknown"
+ iterates, claim = self._source(right, loop, probe, hop=True)
+ if iterates == "unknown" and baseline == "unknown":
+ return None
+ return LoopSource(iterates, claim or baseline)
+
+ def _source(self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ """What *expr* walks, and the magnitude that settles when it differs from
+ what the iterable's syntax alone proves (``None``: keep that).
+
+ *hop* allows one step through the name's last assignment before the loop.
+ """
+ if expr is None:
+ return "unknown", None
+ if expr.type in ("await", "parenthesized_expression"):
+ inner = next((c for c in expr.children if c.is_named), None)
+ return self._source(inner, loop, probe, hop=hop)
+ if expr.type == "boolean_operator" and any(c.type == "or" for c in expr.children):
+ return self._source(expr.child_by_field_name("left"), loop, probe, hop=hop)
+ if expr.type in ("identifier", "attribute") and lex.is_config_name(_text(expr)):
+ # A registry grows with what users register: the name alone bounds nothing.
+ return "config_table", None
+ if expr.type == "identifier":
+ return self._name_source(expr, loop, probe, hop=hop)
+ if expr.type == "attribute":
+ return self._attribute_source(expr, loop, probe, hop=hop)
+ if expr.type == "subscript" and self._slice_magnitude(expr) == "bounded":
+ # A fixed-width slice bounds the trip count, not what is walked.
+ walked = self._source(expr.child_by_field_name("value"), loop, probe, hop=hop)
+ return walked[0], "bounded"
+ if expr.type == "call":
+ return self._call_source(expr, loop, probe, hop=hop)
+ return "unknown", None
+
+ def _attribute_source(self, expr: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ """``result.rows`` walks ``result``; ``self._TABLES`` bound to a literal is a constant."""
+ attr = expr.child_by_field_name("attribute")
+ name = (attr.text if attr is not None else b"") or b""
+ if name.decode("utf-8", "replace") in _PY_RESULT_PROJECTIONS:
+ return self._source(expr.child_by_field_name("object"), loop, probe, hop=hop)
+ if self._literal_constant(expr, name):
+ return "constant", "bounded"
+ return "unknown", None
+
+ def _name_source(self, name: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ rhs = self.reaching_value(loop, name.text or b"") if hop else None
+ if rhs is not None:
+ return self._source(rhs, loop, probe, hop=False)
+ fn = loop.parent
+ while fn is not None and fn.type != "function_definition":
+ fn = fn.parent
+ params = fn.child_by_field_name("parameters") if fn is not None else None
+ if params is not None and _text(name) in self._param_names(params):
+ return "parameter", None
+ return "unknown", None
+
+ def _call_source(self, call: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ method, fn, receiver, args = self._call_bits(call)
+ if lex.is_config_name(self._dotted_path(fn) or "", callee=True):
+ return "config_table", None
+ if method == "range" and receiver is None:
+ return self._range_source(call, args, loop, probe, hop=hop)
+ walked = self._walked_arg_or_receiver(method, receiver, args)
+ if walked is not None:
+ return self._source(walked, loop, probe, hop=hop)
+ if method in _PY_LISTINGS or method in _PY_STREAMS or (method == "open" and not receiver):
+ return self._file_source(call, method, loop)
+ if probe(call) in ("db", "network"):
+ return self._query(call, loop)
+ if receiver is not None and method in _PY_RESULT_PROJECTIONS:
+ return self._source(receiver, loop, probe, hop=hop)
+ return "unknown", None
+
+ def _call_bits(self, call: Node) -> tuple[str, Node | None, Node | None, list[Node]]:
+ """``(method, callee, receiver, positional args)`` of a call."""
+ fn = call.child_by_field_name("function")
+ receiver = fn.child_by_field_name("object") if fn is not None and fn.type == "attribute" else None
+ arg_list = call.child_by_field_name("arguments")
+ args = [c for c in arg_list.children if c.is_named] if arg_list is not None else []
+ return self.callee_method_name(call) or "", fn, receiver, args
+
+ @staticmethod
+ def _walked_arg_or_receiver(
+ method: str, receiver: Node | None, args: list[Node]
+ ) -> Node | None:
+ """``sorted(xs)`` / ``enumerate(xs)`` walk ``xs``; ``d.items()`` walks ``d``."""
+ if method in _PY_PASS_THROUGH and args and args[0].type != "keyword_argument":
+ return args[0]
+ if receiver is not None and method in ("items", "values", "keys"):
+ return receiver
+ return None
+
+ def _file_source(self, call: Node, method: str, loop: Node) -> Source:
+ """A directory listing (grows when it walks the repository tree) or a stream."""
+ if method in _PY_LISTINGS:
+ return "listing", "grows_with_data" if self._walks_repo(call, method, loop) else None
+ return "stream", None
+
+ def _range_source(
+ self, call: Node, args: list[Node], loop: Node, probe: SinkProbe, *, hop: bool
+ ) -> Source:
+ """``range(3)`` is a constant; ``range(0, len(ids), N)`` walks ``ids``."""
+ if self._range_magnitude(call, loop, probe, hop=hop) == "bounded":
+ return "constant", "bounded"
+ stop = args[1] if len(args) > 1 else (args[0] if args else None)
+ if stop is not None and stop.type == "call" and self.callee_method_name(stop) == "len":
+ inner = [c for c in stop.child_by_field_name("arguments").children if c.is_named]
+ if len(inner) == 1:
+ return self._source(inner[0], loop, probe, hop=hop)
+ return "unknown", None
+
+ def reads_config_table(self, call: Node, nodes: list[Node]) -> bool:
+ """Every SQL text and selected model names a registry or configuration table."""
+ walked = [n for part in nodes for n in self._walk(part)]
+ sql = [
+ n.text or b""
+ for n in walked
+ if n.type == "string" and _PY_SQL_LITERAL_RE.match(_text(n))
+ ]
+ models = [
+ model
+ for n in walked
+ if n.type == "call" and self.callee_method_name(n) in ("select", "query")
+ for model in self._selected_models(n)
+ ]
+ return bool(sql or models) and (
+ all(lex.is_config_sql(text) for text in sql)
+ and all(lex.is_config_model(model) for model in models)
+ )
+
+ def _selected_models(self, select: Node) -> list[str]:
+ """``select(Repository)`` / ``query(Repository.id, Repository.path)`` -> the model
+ names; ``func.count()`` and other expressions name none."""
+ args = select.child_by_field_name("arguments")
+ models: list[str] = []
+ for arg in (c for c in args.children if c.is_named) if args is not None else ():
+ path = self._dotted_path(arg) or ""
+ head = next((seg for seg in path.split(".") if _PY_MODEL_NAME_RE.match(seg)), None)
+ if head is not None:
+ models.append(head)
+ return models
+
+ def _literal_constant(self, expr: Node, name: bytes) -> bool:
+ """``self._TABLES`` where the file binds ``_TABLES`` once, to a literal collection."""
+ text = name.decode("utf-8", "replace")
+ if len(text.strip("_")) < 2 or not text.isupper():
+ return False
+ root = expr
+ while root.parent is not None:
+ root = root.parent
+ rights = [
+ n.child_by_field_name("right")
+ for n in self._walk(root)
+ if n.type == "assignment"
+ and (left := n.child_by_field_name("left")) is not None
+ and left.type == "identifier"
+ and left.text == name
+ ]
+ return (
+ len(rights) == 1
+ and rights[0] is not None
+ and rights[0].type in _PY_CONSTANT_RHS
+ and all(_scalar_literal(item) for item in rights[0].named_children)
+ )
def _magnitude(
self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool
@@ -947,7 +1175,7 @@ class PythonPerfDialect(BasePerfDialect):
inner = next((c for c in expr.children if c.is_named), None)
return self._magnitude(inner, loop, probe, hop=hop)
if expr.type == "identifier" and hop and expr.text:
- rhs = self._reaching_rhs(loop, expr.text)
+ rhs = self.reaching_value(loop, expr.text)
return self._magnitude(rhs, loop, probe, hop=False)
if expr.type == "subscript":
return self._slice_magnitude(expr)
@@ -973,15 +1201,15 @@ class PythonPerfDialect(BasePerfDialect):
# A read capped in the query is as large as its cap; a statement or result
# built before the loop (``stmt = ....limit(n)``) is read one assignment back.
names = [n.text for n in self._walk(expr) if n.type == "identifier"]
- parts = [expr, *filter(None, (self._reaching_rhs(loop, name) for name in names))]
- capped = any(self._caps_read(n, loop) for part in parts for n in self._walk(part))
+ parts = [expr, *filter(None, (self.reaching_value(loop, name) for name in names))]
+ capped = any(self.is_cap(n, loop) for part in parts for n in self._walk(part))
return None if capped else "grows_with_data"
if member and method in _PY_RESULT_PROJECTIONS:
receiver = expr.child_by_field_name("function").child_by_field_name("object")
return self._grows(self._magnitude(receiver, loop, probe, hop=hop))
return None
- def _caps_read(self, node: Node, loop: Node) -> bool:
+ def is_cap(self, node: Node, loop: Node) -> bool:
"""``.limit(n)`` and peers, a SQL string saying ``LIMIT n``, or ``.in_()`` over
one chunk of keys."""
if node.type == "string":
@@ -1043,23 +1271,6 @@ class PythonPerfDialect(BasePerfDialect):
return "bounded"
return None
- def _reaching_rhs(self, loop: Node, name: bytes) -> Node | None:
- """The right side of the last ``name = ...`` before *loop* in its function."""
- scope = loop.parent
- while scope is not None and scope.type != "function_definition":
- scope = scope.parent
- if scope is None:
- return None
- last: Node | None = None
- for node in self._walk(scope, prune=_PY_SCOPES):
- if node.type != "assignment" or node.end_byte > loop.start_byte:
- continue
- left = node.child_by_field_name("left")
- named = left is not None and left.type == "identifier" and left.text == name
- if named and (last is None or node.start_byte > last.start_byte):
- last = node
- return last.child_by_field_name("right") if last is not None else None
-
def batch_form(self, sink: Node, loop: Node, probe: SinkProbe) -> BatchForm | None:
target = loop.child_by_field_name("left")
if loop.type != "for_statement" or target is None or target.type != "identifier":
@@ -1124,9 +1335,10 @@ class PythonPerfDialect(BasePerfDialect):
builtin (``str(r.id)``) or a method on a local (``res.first()``)."""
if not self.callee_is_attribute(call):
return not (assigned and self.callee_method_name(call) in _PY_BUILTINS)
- rhs = self._reaching_rhs(call, (self.callee_root_name(call) or "").encode())
- if rhs is None:
+ before = self.bindings_before(call, (self.callee_root_name(call) or "").encode())
+ if not before:
return True
+ rhs = before[-1][1]
return not (assigned or self._rhs_is_list(rhs) or self._rhs_is_nonlist_container(rhs))
def _limited_downstream(self, sink: Node, body: Node) -> bool:
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/ruby.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/ruby.py
index db9e7d7a4..ab81eeecd 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/ruby.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/ruby.py
@@ -31,7 +31,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-from .base import BasePerfDialect
+from .iterables import IterableSources
from .python import HTTP_VERBS
if TYPE_CHECKING:
@@ -246,7 +246,45 @@ def _decode(node: Node | None) -> str | None:
return node.text.decode("utf-8", "replace")
-class RubyPerfDialect(BasePerfDialect):
+_RB_FN_KINDS: frozenset[str] = frozenset({"method", "singleton_method", "lambda", "program"})
+# ActiveRecord batch iteration: the call reads the table the block walks.
+_AR_BATCHES: frozenset[str] = frozenset({"find_each", "find_in_batches", "in_batches"})
+# Relation builders: on a model class, walking the result reads its table.
+_AR_RELATION: frozenset[str] = _AR_BATCHES | frozenset(
+ {
+ "all", "where", "order", "reorder", "includes", "preload", "eager_load", "joins",
+ "left_joins", "left_outer_joins", "select", "distinct", "group", "having", "not",
+ "unscoped", "references", "rewhere", "limit", "offset",
+ }
+)
+# Constants that are core or stdlib classes, never a model.
+_RB_CORE_CONSTANTS: frozenset[str] = frozenset(
+ {"File", "Dir", "IO", "Time", "Date", "DateTime", "JSON", "YAML", "Set", "Hash", "Array",
+ "String", "Struct", "Kernel", "Process", "Thread", "Rails", "ENV", "ObjectSpace"}
+)
+
+
+def _is_model_name(node: Node) -> bool:
+ """``User`` / ``Admin::User``: a CapWords class that is not core Ruby."""
+ if node.type == "scope_resolution":
+ node = node.child_by_field_name("name") or node
+ name = _decode(node) or ""
+ return (
+ node.type == "constant"
+ and name not in _RB_CORE_CONSTANTS
+ and any(c.islower() for c in name)
+ )
+
+
+def _rb_bindings(node: Node) -> list[tuple[bytes, Node]]:
+ """What *node* binds: ``x = v`` or ``x ||= v``."""
+ if node.type not in ("assignment", "operator_assignment"):
+ return []
+ left, right = node.child_by_field_name("left"), node.child_by_field_name("right")
+ ok = left is not None and left.type in ("identifier", "instance_variable") and right is not None
+ return [(left.text or b"", right)] if ok else []
+
+class RubyPerfDialect(IterableSources):
language = "ruby"
markers = frozenset(
{
@@ -513,6 +551,108 @@ class RubyPerfDialect(BasePerfDialect):
return "lock_in_loop"
return None
+ # -- what a loop walks (perf/dialects/iterables.py) ---------------------------
+
+ function_kinds = _RB_FN_KINDS
+ change_kinds = frozenset({"assignment", "operator_assignment", "binary"})
+ grow_methods = frozenset({"push", "append", "concat", "insert", "store", "unshift"})
+ conditional_kinds = frozenset(
+ {
+ "if", "unless", "elsif", "else", "case", "when", "in_clause", "while", "until",
+ "for", "begin", "rescue", "if_modifier", "unless_modifier", "while_modifier",
+ "until_modifier", "rescue_modifier", "conditional", "block", "do_block",
+ }
+ )
+
+ literal_kinds = frozenset({"array", "string_array", "symbol_array", "hash"})
+ int_kinds = frozenset({"integer"})
+ name_kinds = frozenset({"identifier", "instance_variable"})
+ wrapper_kinds = frozenset({"parenthesized_statements"})
+ pass_through = frozenset(
+ {
+ "map", "flat_map", "collect", "select", "filter", "filter_map", "reject",
+ "sort", "sort_by", "uniq", "compact", "flatten", "to_a", "values", "keys",
+ "reverse", "lazy", "with_index", "each_with_index", "each_slice", "zip",
+ "group_by", "includes", "preload", "eager_load", "order", "distinct",
+ }
+ )
+ caps = frozenset({"first", "take", "last", "sample", "limit"})
+ size_names = frozenset({"size", "length", "count"})
+ listings = frozenset({"glob", "entries", "children", "each_child"})
+ streams = frozenset({"foreach", "readlines", "each_line", "lines"})
+ query_caps = frozenset({"limit", "first", "take"})
+
+ def loop_header(self, loop: Node) -> tuple[str, Node | None]:
+ if loop.type == "for":
+ return "each", self._for_iterable(loop)
+ if loop.type != "call" or self.block_loop_body(loop) is None:
+ return "each", None
+ method = _decode(loop.child_by_field_name("method")) or ""
+ receiver = loop.child_by_field_name("receiver")
+ if method == "times":
+ return "counted", receiver
+ if method == "upto":
+ args = loop.child_by_field_name("arguments")
+ return "counted", args.named_children[0] if args and args.named_children else None
+ if method in _AR_BATCHES or method in self.streams or method in self.listings:
+ return "each", loop # the call itself reads what the block walks
+ if method in _COUNTING_METHODS or method == "loop":
+ return "each", None
+ return "each", receiver
+
+ def call_parts(self, node: Node) -> tuple[str, Node | None, list[Node]] | None:
+ if node.type == "element_reference":
+ # ``Dir["**/*.rb"]`` is ``Dir.glob``.
+ obj = node.child_by_field_name("object")
+ if obj is None or obj.text != b"Dir":
+ return None
+ return "glob", obj, [c for c in node.named_children if c != obj]
+ if node.type != "call":
+ return None
+ args = node.child_by_field_name("arguments")
+ return (
+ _decode(node.child_by_field_name("method")) or "",
+ node.child_by_field_name("receiver"),
+ [c for c in args.named_children if c.type != "block_argument"] if args else [],
+ )
+
+ def is_literal(self, node: Node) -> bool:
+ if node.type == "range":
+ return self._is_constant_collection(node)
+ return super().is_literal(node)
+
+ def query_chain(self, node: Node) -> bool:
+ """``Order.where(x)`` / ``User.includes(:posts).order(:id)``: a relation
+ built on a model class, which reads its table when walked."""
+ relation = False
+ cur: Node | None = node
+ while cur is not None and cur.type == "call":
+ method = _decode(cur.child_by_field_name("method")) or ""
+ if method not in _AR_RELATION:
+ return False
+ relation = True
+ cur = cur.child_by_field_name("receiver")
+ return relation and cur is not None and _is_model_name(cur)
+
+ def bindings_of(self, node: Node) -> list[tuple[bytes, Node]]:
+ return _rb_bindings(node)
+
+ def mutated_name(self, node: Node) -> bytes | None:
+ # Of the binary operators only ``xs << x`` grows its left side.
+ if node.type == "binary" and _decode(node.child_by_field_name("operator")) != "<<":
+ return None
+ return super().mutated_name(node)
+
+ def param_names(self, loop: Node) -> set[bytes]:
+ scope = self._scope(loop)
+ params = scope.child_by_field_name("parameters") if scope is not None else None
+ names: set[bytes] = set()
+ for param in params.named_children if params is not None else ():
+ name = param if param.type == "identifier" else param.child_by_field_name("name")
+ if name is not None and name.text:
+ names.add(name.text)
+ return names
+
def is_lock_scope(self, node: Node) -> bool:
# ``mutex.synchronize do … end`` — the block argument is the held
# region (the walker raises lock_depth for block-typed children only).
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py
index e3f1e81fe..b7adf2fcb 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py
@@ -9,14 +9,18 @@ from __future__ import annotations
import re
from typing import TYPE_CHECKING, ClassVar
-from ..loop_facts import BatchForm
+from .. import loop_lexicon as lex
+from ..loop_facts import BatchForm, LoopSource
+from .iterables import IterableSources
from .python import HTTP_VERBS
from .ts_js_markers import TsJsMarkerHooks, first_named_child, identifier_name
if TYPE_CHECKING:
from tree_sitter import Node
- from ..loop_facts import LoopMagnitude, SinkProbe
+ from ..loop_facts import LoopIterates, LoopMagnitude, SinkProbe
+
+ Source = tuple[LoopIterates, LoopMagnitude | None]
# TypeScript / JavaScript. Only DISTINCTIVE method names are trusted without
# import resolution: bare create/update/delete/count/exec collide hard with
@@ -212,6 +216,51 @@ def _module_number_constant(name: bytes, node: Node) -> bool:
return cur is not None and _top_level_number_const(cur, name)
+# Directory listings; a ``walk*`` helper or a ``recursive``/``**`` argument walks a tree.
+_TS_LISTINGS = frozenset(
+ {"readdir", "readdirSync", "glob", "globSync", "walk", "walkSync", "walkFiles"}
+)
+# Array and Map methods that walk (a projection of) their receiver.
+_TS_PASS_THROUGH = frozenset(
+ {"filter", "map", "flatMap", "toSorted", "sort", "toReversed", "reverse", "entries",
+ "keys", "values"}
+)
+# ``Object.entries(x)`` / ``Array.from(x)`` walk their first argument.
+_TS_ARG_WALKERS = frozenset({"entries", "keys", "values", "from"})
+
+
+def _text(node: Node) -> str:
+ return (node.text or b"").decode("utf-8", "replace")
+
+
+def _param_names(fn: Node) -> set[bytes]:
+ params = fn.child_by_field_name("parameters") or fn.child_by_field_name("parameter")
+ if params is None:
+ return set()
+ kinds = ("identifier", "shorthand_property_identifier_pattern")
+ return {n.text for n in _ts_walk(params) if n.type in kinds and n.text}
+
+
+def _counted_stop(loop: Node) -> Node | None:
+ """``n`` of a counted loop's ``i < n`` / ``i <= n``."""
+ cond = loop.child_by_field_name("condition")
+ if cond is not None and cond.type == "expression_statement":
+ cond = first_named_child(cond)
+ if cond is None or cond.type != "binary_expression":
+ return None
+ if not any(c.type in ("<", "<=") for c in cond.children):
+ return None
+ return cond.child_by_field_name("right")
+
+
+def _fixed_count(stop: Node) -> bool:
+ """``3``, or ``MAX`` that its file binds once to a number."""
+ if stop.type == "number":
+ return True
+ name = identifier_name(stop)
+ return bool(name) and name.isupper() and _module_number_constant(stop.text or b"", stop)
+
+
def _slice_is_bounded(call: Node) -> bool:
"""``.slice(a, N)`` where N is an integer literal or an ALL_CAPS constant
its own file binds to one: a constant-width read, not the whole collection.
@@ -250,7 +299,7 @@ def _assigned_value(node: Node, name: bytes) -> Node | None:
return value
-class TsJsPerfDialect(TsJsMarkerHooks):
+class TsJsPerfDialect(TsJsMarkerHooks, IterableSources):
language = "typescript"
markers = frozenset(
{
@@ -432,26 +481,148 @@ class TsJsPerfDialect(TsJsMarkerHooks):
return self._grows(self._magnitude_of_expr(_callee_object(call), probe, depth + 1))
return None
- def _reaching_assignment(self, loop: Node, name: bytes) -> Node | None:
- """The value of the last ``name = ...`` before *loop* in its function."""
- last: tuple[int, Node] | None = None
- for node in self._walk(_enclosing_scope(loop), prune=_FN_KINDS):
- if node.end_byte > loop.start_byte:
- continue
- value = _assigned_value(node, name)
- if value is not None and (last is None or node.start_byte > last[0]):
- last = (node.start_byte, value)
- return last[1] if last is not None else None
-
- def loop_magnitude(self, loop: Node, probe: SinkProbe) -> LoopMagnitude | None:
+ # -- what a loop walks: the shared reading (perf/dialects/iterables.py) -------
+ function_kinds = _FN_KINDS
+ member_kinds = frozenset({"member_expression"})
+ recursive_listings = frozenset({"walk", "walkSync", "walkFiles"})
+ recursive_marks = (b"**", b"recursive: true")
+ conditional_kinds = frozenset(
+ {
+ "if_statement", "else_clause", "switch_statement", "switch_case", "switch_default",
+ "for_statement", "for_in_statement", "while_statement", "do_statement",
+ "try_statement", "catch_clause", "ternary_expression",
+ }
+ )
+ # The iterable's own syntax (``_magnitude_of_expr``) says whether a read grows.
+ read_grows = None
+
+ def _scope(self, anchor: Node) -> Node | None:
+ # A script's top-level loop reads the bindings around it.
+ return _enclosing_scope(anchor)
+
+ def bindings_of(self, node: Node) -> list[tuple[bytes, Node]]:
+ fields = _ASSIGNMENT_FIELDS.get(node.type)
+ if fields is None:
+ return []
+ target, value = (node.child_by_field_name(field) for field in fields)
+ ok = target is not None and target.type == "identifier" and target.text and value is not None
+ return [(target.text, value)] if ok else []
+
+ def loop_source(self, loop: Node, probe: SinkProbe) -> LoopSource | None:
+ if loop.type == "for_statement":
+ iterates, claim = self._counted_source(loop, probe)
+ return LoopSource(iterates, claim or "unknown") if iterates != "unknown" else None
if not self.is_iteration_loop(loop):
return None
right = loop.child_by_field_name("right")
- magnitude = self._magnitude_of_expr(right, probe)
- if magnitude is None and right is not None and identifier_name(right):
- return self._magnitude_of_expr(self._reaching_assignment(loop, right.text), probe)
+ baseline = self._expr_magnitude(right, loop, probe)
+ iterates, claim = self._source(right, loop, probe, hop=True)
+ if iterates == "unknown" and baseline is None:
+ return None
+ return LoopSource(iterates, claim or baseline or "unknown")
+
+ def _expr_magnitude(
+ self, expr: Node | None, loop: Node, probe: SinkProbe
+ ) -> LoopMagnitude | None:
+ """What *expr*'s syntax proves, one step through the name's last assignment."""
+ magnitude = self._magnitude_of_expr(expr, probe)
+ if magnitude is None and expr is not None and identifier_name(expr):
+ return self._magnitude_of_expr(self.reaching_value(loop, expr.text), probe)
return magnitude
+ def _counted_source(self, loop: Node, probe: SinkProbe) -> Source:
+ """``i < 3`` / ``i < MAX`` (a literal its file binds once) is a constant;
+ ``i < rows.length`` walks ``rows``."""
+ stop = _counted_stop(loop)
+ if stop is None:
+ return "unknown", None
+ if _fixed_count(stop):
+ return "constant", "bounded"
+ prop = stop.child_by_field_name("property") if stop.type == "member_expression" else None
+ if prop is not None and prop.text == b"length":
+ walked = stop.child_by_field_name("object")
+ iterates, claim = self._source(walked, loop, probe, hop=True)
+ return iterates, claim or self._expr_magnitude(walked, loop, probe)
+ return "unknown", None
+
+ def _source(self, expr: Node | None, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ """What *expr* walks, and the magnitude that settles when it differs from
+ what the iterable's syntax alone proves (``None``: keep that)."""
+ if expr is None:
+ return "unknown", None
+ node = _unwrap(expr)
+ if node.type == "binary_expression" and any(c.type in ("||", "??") for c in node.children):
+ return self._source(node.child_by_field_name("left"), loop, probe, hop=hop)
+ if node.type in ("identifier", "member_expression") and lex.is_config_name(_text(node)):
+ # A registry grows with what users register: the name alone bounds nothing.
+ return "config_table", None
+ if node.type == "identifier":
+ value = self.reaching_value(loop, node.text or b"") if hop else None
+ if value is not None:
+ return self._source(value, loop, probe, hop=False)
+ if (node.text or b"") in _param_names(_enclosing_scope(loop)):
+ return "parameter", None
+ return "unknown", None
+ if node.type == "member_expression" and _is_data_projection(node):
+ return self._source(node.child_by_field_name("object"), loop, probe, hop=hop)
+ if node.type == "call_expression":
+ return self._call_source(node, loop, probe, hop=hop)
+ return "unknown", None
+
+ def _call_source(self, call: Node, loop: Node, probe: SinkProbe, *, hop: bool) -> Source:
+ method, fn, receiver, first = self._call_bits(call)
+ if lex.is_config_name(self._dotted_path(fn) or "", callee=True):
+ return "config_table", None
+ if method == "slice" and _slice_is_bounded(call):
+ # A fixed-width slice bounds the trip count, not what is walked.
+ return self._source(receiver, loop, probe, hop=hop)[0], "bounded"
+ if receiver is not None and (
+ receiver.text in (b"Object", b"Array") or method in _TS_PASS_THROUGH
+ ):
+ return self._walked_source(method, receiver, first, loop, probe, hop=hop)
+ if method in _TS_LISTINGS:
+ return "listing", "grows_with_data" if self._walks_repo(call, method, loop) else None
+ if probe(call) in ("db", "network"):
+ return self._query(call, loop)
+ return "unknown", None
+
+ def _call_bits(self, call: Node) -> tuple[str, Node | None, Node | None, Node | None]:
+ """``(method, callee, receiver, first argument)`` of a call."""
+ fn = call.child_by_field_name("function")
+ is_member = fn is not None and fn.type == "member_expression"
+ args = call.child_by_field_name("arguments")
+ first = next((c for c in args.children if c.is_named), None) if args is not None else None
+ return self.callee_method_name(call) or "", fn, _callee_object(call) if is_member else None, first
+
+ def _walked_source(
+ self,
+ method: str,
+ receiver: Node,
+ first: Node | None,
+ loop: Node,
+ probe: SinkProbe,
+ *,
+ hop: bool,
+ ) -> Source:
+ """``Object.entries(x)`` / ``Array.from(x)`` walk ``x``; ``xs.filter(f)`` walks ``xs``."""
+ if receiver.text in (b"Object", b"Array"):
+ if method not in _TS_ARG_WALKERS:
+ return "unknown", None
+ return self._source(first, loop, probe, hop=hop)
+ return self._source(receiver, loop, probe, hop=hop)
+
+ def reads_config_table(self, call: Node, nodes: list[Node]) -> bool:
+ """Every table the read names is a registry (``FROM repos``), or with no SQL,
+ the delegate does (``prisma.repository.findMany()``)."""
+ sql = [n.text or b"" for n in self._walk(call) if n.type in ("string", "template_string")]
+ sql = [text for text in sql if lex.sql_tables(text)]
+ if sql:
+ return all(lex.is_config_sql(text) for text in sql)
+ model = _callee_object(call)
+ prop = model.child_by_field_name("property") if model is not None else None
+ delegate = _text(prop) if prop is not None and model.type == "member_expression" else ""
+ return bool(delegate) and lex.is_config_model(delegate[:1].upper() + delegate[1:])
+
def _awaited_wrapper_call(self, closure: Node) -> Node | None:
"""The awaited call *closure* is passed to (``await limit(() => f(x))``).
diff --git a/packages/core/src/repowise/core/analysis/health/perf/facts.py b/packages/core/src/repowise/core/analysis/health/perf/facts.py
index 28349c4b2..df3a7348c 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/facts.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/facts.py
@@ -155,6 +155,7 @@ def evidence_row(facts: ObservationFacts) -> dict[str, Any]:
**({"loop_line": loop} if (loop := facts.details.get("loop_line")) else {}),
**({"sink_call": call} if (call := facts.details.get("sink_call")) else {}),
**({"subject": subject} if (subject := facts.details.get("subject")) else {}),
+ **({"loop_iterates": what} if (what := facts.details.get("loop_iterates")) else {}),
}
diff --git a/packages/core/src/repowise/core/analysis/health/perf/loop_facts.py b/packages/core/src/repowise/core/analysis/health/perf/loop_facts.py
index 70771959b..d1e2219b9 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/loop_facts.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/loop_facts.py
@@ -1,10 +1,11 @@
"""What a loop around a performance hit proves about the fix.
A hit says the cost is paid per iteration. These facts say how many iterations
-to expect and whether the obvious fix is provably the same program: a bulk
-counterpart that takes every key at once, or a bound the fan-out would run
-under. Each is read off the syntax of one function by a language dialect, and
-each answers ``None`` or ``unknown`` when that syntax does not settle it.
+to expect, what the loop walks, and whether the obvious fix is provably the
+same program: a bulk counterpart that takes every key at once, or a bound the
+fan-out would run under. Each is read off the syntax of one function by a
+language dialect, and each answers ``None`` or ``unknown`` when that syntax
+does not settle it.
"""
from __future__ import annotations
@@ -18,10 +19,43 @@ if TYPE_CHECKING:
LoopMagnitude = Literal["grows_with_data", "bounded", "unknown"]
+# What a loop walks. ``query``: rows a database or network read returned;
+# ``parameter``: a collection the caller passed in; ``caller_key``: the rows of
+# one primary key; ``rare_subset``: rows of a growing table a filter keeps
+# only for a rare state (expired, stale, failed); ``config_table``: a registry
+# or configuration table (registered repos, providers, plugins, settings);
+# ``constant``: a literal, a constant bound or a fixed-width slice;
+# ``listing``: a directory listing; ``stream``: lines or chunks read off a
+# file, a process or a response.
+LoopIterates = Literal[
+ "query",
+ "parameter",
+ "caller_key",
+ "rare_subset",
+ "config_table",
+ "constant",
+ "listing",
+ "stream",
+ "unknown",
+]
+
+# Why splitting a loop's per-iteration calls into one batch would change the
+# program: ``first_match`` stops at the first call that answers;
+# ``turn_loop`` feeds each call's result into the next call.
+NotBatchable = Literal["first_match", "turn_loop"]
+
# The boundary kind of a call node when it is an I/O sink, else ``None``.
SinkProbe = Callable[["Node"], str | None]
+@dataclass(frozen=True, slots=True)
+class LoopSource:
+ """What one loop's iterable is, and how many iterations that proves."""
+
+ iterates: LoopIterates = "unknown"
+ magnitude: LoopMagnitude = "unknown"
+
+
@dataclass(frozen=True, slots=True)
class BatchForm:
"""The bulk counterpart of a per-key sink, written as the code would call it.
@@ -43,6 +77,9 @@ class LoopFacts:
concurrency_bound: str | None = None
# No element or index the loop binds reaches the sink call: nothing per-key to batch.
key_unused: bool = False
+ # What the innermost loop walks, and why its calls cannot be one batch.
+ iterates: LoopIterates = "unknown"
+ not_batchable: NotBatchable | None = None
def as_details(self) -> dict[str, Any]:
"""Finding ``details`` keys; defaults are omitted so old findings are unchanged."""
@@ -58,7 +95,29 @@ class LoopFacts:
details["concurrency_bound"] = self.concurrency_bound
if self.key_unused:
details["loop_key_unused"] = True
+ if self.iterates != "unknown":
+ details["loop_iterates"] = self.iterates
+ if self.not_batchable:
+ details["loop_not_batchable"] = self.not_batchable
return details
-__all__ = ["BatchForm", "LoopFacts", "LoopMagnitude", "SinkProbe"]
+def combined_magnitude(sources: list[LoopSource]) -> LoopMagnitude:
+ """Grows if any enclosing loop grows; bounded only if every one is, since a
+ retry loop inside a loop over rows still runs once per row."""
+ found = {source.magnitude for source in sources}
+ if "grows_with_data" in found:
+ return "grows_with_data"
+ return "bounded" if found == {"bounded"} else "unknown"
+
+
+__all__ = [
+ "BatchForm",
+ "LoopFacts",
+ "LoopIterates",
+ "LoopMagnitude",
+ "LoopSource",
+ "NotBatchable",
+ "SinkProbe",
+ "combined_magnitude",
+]
diff --git a/packages/core/src/repowise/core/analysis/health/perf/loop_lexicon.py b/packages/core/src/repowise/core/analysis/health/perf/loop_lexicon.py
new file mode 100644
index 000000000..1370f2154
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/perf/loop_lexicon.py
@@ -0,0 +1,186 @@
+"""Names and query text that say what a loop walks, shared by every dialect.
+
+A dialect finds the iterable; these answer whether its name, ORM model or SQL
+text names a registry or configuration table (what is walked, not how many:
+such tables grow with use), the rows of one primary key, a filter that keeps
+only a rare state of a growing table, a read capped in its SQL, or a
+repository root a listing walks. Each answers ``False`` when the text does
+not settle it.
+"""
+
+from __future__ import annotations
+
+import re
+from itertools import pairwise
+
+# A collection a user registers or configures, as the last word of its name.
+_CONFIG_NOUNS = frozenset(
+ {
+ "repos",
+ "repositories",
+ "workspaces",
+ "providers",
+ "plugins",
+ "settings",
+ "registry",
+ "registries",
+ }
+)
+# A function that returns repos or workspaces may be scanning a disk for them
+# (``discover_repos(root)``); only these nouns name configuration as a callee.
+_CALLEE_NOUNS = _CONFIG_NOUNS - {"repos", "repositories", "workspaces"}
+# Containers whose members are configuration wherever they sit in a path
+# (``ws_config.repos``, ``settings.get("exclude")``), unless what is listed is
+# files. A bare ``cfg`` is as often a control-flow graph, so only ``x_cfg`` counts.
+_CONFIG_CONTAINERS = frozenset({"config", "settings", "registry"})
+_CFG_SUFFIX = re.compile(r"[a-z0-9](?:_cfg|Cfg)\b")
+# A scan of text yields matches or lines of it, whatever the receiver is named.
+_TEXT_SCANS = frozenset({"finditer", "findall", "split", "rsplit", "splitlines"})
+# ``provider_dirs`` / ``repo_paths``: one config noun, then what is listed of it.
+_CONFIG_SINGULARS = frozenset({"repo", "repository", "workspace", "provider", "plugin"})
+_CALLEE_SINGULARS = _CONFIG_SINGULARS - {"repo", "repository", "workspace"}
+_CONFIG_LISTED = frozenset({"ids", "names", "paths", "dirs", "roots", "keys", "entries"})
+# ``page_types`` spans two words.
+_CONFIG_PAIRS = frozenset({("page", "types")})
+
+_CONFIG_MODELS = frozenset(
+ {
+ "Repository",
+ "Repo",
+ "Workspace",
+ "WorkspaceRepo",
+ "WorkspaceRepository",
+ "Provider",
+ "Plugin",
+ "Setting",
+ "Settings",
+ "Config",
+ "Configuration",
+ "PageType",
+ "Registry",
+ }
+)
+_CONFIG_TABLES = frozenset(
+ {
+ "repositories",
+ "repos",
+ "workspaces",
+ "workspace_repos",
+ "providers",
+ "plugins",
+ "settings",
+ "config",
+ "configs",
+ "page_types",
+ }
+)
+
+_WORD = re.compile(r"[A-Z]+(?![a-z])|[A-Z]?[a-z]+|\d+")
+_SQL_FROM = re.compile(rb"\b(?:FROM|JOIN)\s+[\"`\[]?([A-Za-z_][\w.]*)", re.IGNORECASE)
+# Several tables or a recursive walk: no longer the rows of one key.
+_SQL_WIDENS = re.compile(rb"\b(?:JOIN|UNION|RECURSIVE|IN)\b|\bin_\(|\.join\(", re.IGNORECASE)
+# ``WHERE id = ?`` / ``t.id = :id`` / ``Model.id == key``; ``parent_id = ?`` is not.
+_KEY_EQUALITY = re.compile(
+ rb"(?<![\w.])(?:\w+\.)?id\s*(?:=\s*(?:\?|:\w+|%s|%\(\w+\)s|\$\d+)|==\s*(?!=)\w)"
+)
+# A time threshold on an expiry-like column, or a rare-state literal.
+_RARE_FILTER = re.compile(
+ rb"\w*(?:expire|expiry|deadline|heartbeat|lease|ttl|timeout)\w*\s*<"
+ rb"|[\"'](?:stale|expired|failed|crashed|orphan|orphaned|dead|zombie|timed_out"
+ rb"|abandoned|stuck)[\"']",
+ re.IGNORECASE,
+)
+_ROOT_OWNERS = frozenset({"repo", "repository", "project", "workspace"})
+_ROOT_WORDS = frozenset({"root", "path", "dir"})
+
+
+def words(name: str) -> list[str]:
+ """Lower-cased words of a name: ``ws_config.repos`` -> ws, config, repos."""
+ return [w.lower() for part in re.split(r"[^A-Za-z0-9]+", name) for w in _WORD.findall(part)]
+
+
+def is_config_name(name: str, *, callee: bool = False) -> bool:
+ """A name (``ws_config.repos``, ``_iter_provider_dirs``) for a configured collection.
+
+ *callee*: the name is a function the loop calls for its collection.
+ """
+ ws = words(name)
+ if not ws or ws[-1] in _TEXT_SCANS:
+ return False
+ if ws[-1] in (_CALLEE_NOUNS if callee else _CONFIG_NOUNS):
+ return True
+ if ws[-1] != "files" and (_CONFIG_CONTAINERS.intersection(ws) or _CFG_SUFFIX.search(name)):
+ return True
+ if len(ws) >= 2 and (ws[-2], ws[-1]) in _CONFIG_PAIRS:
+ return True
+ singulars = _CALLEE_SINGULARS if callee else _CONFIG_SINGULARS
+ return len(ws) >= 2 and ws[-1] in _CONFIG_LISTED and ws[-2] in singulars
+
+
+def is_config_model(name: str) -> bool:
+ """An ORM model (last dotted segment) that is a registry or configuration table."""
+ return name.rsplit(".", 1)[-1] in _CONFIG_MODELS
+
+
+def sql_tables(sql: bytes) -> list[str]:
+ """The tables a SQL text reads, lower-cased, schema dropped."""
+ return [m.group(1).decode().rsplit(".", 1)[-1].lower() for m in _SQL_FROM.finditer(sql)]
+
+
+def is_config_sql(sql: bytes) -> bool:
+ tables = sql_tables(sql)
+ return bool(tables) and all(t in _CONFIG_TABLES for t in tables)
+
+
+def is_key_lookup(text: bytes) -> bool:
+ """The read filters on equality with one primary key and nothing widens it."""
+ return bool(_KEY_EQUALITY.search(text)) and not _SQL_WIDENS.search(text)
+
+
+def is_rare_filter(text: bytes) -> bool:
+ return bool(_RARE_FILTER.search(text))
+
+
+# ``authority_id = ?`` / ``where(user_id: u.id)`` / ``findAllByOwnerId(id)``: the
+# rows that belong to one parent row.
+_PARENT_KEY = re.compile(
+ rb"\b[a-z]\w*_id\b(?:\s*=\s*(?:\?|:\w+|@\w+|\$\d+)|:\s*[^\s:])"
+ rb"|\b(?:find|get|query|read|stream|count)\w*By\w+Id\s*\(",
+ re.IGNORECASE,
+)
+
+
+def is_parent_key_filter(text: bytes) -> bool:
+ """The read filters on equality with one foreign key."""
+ return bool(_PARENT_KEY.search(text))
+
+
+_SQL_LIMIT = re.compile(
+ rb"\bselect\b.*?(?:\blimit\s+\d+\b|\btop\s*\(?\s*\d+|\bfetch\s+first\s+\d+)",
+ re.IGNORECASE | re.DOTALL,
+)
+
+
+def has_sql_limit(text: bytes) -> bool:
+ """A SELECT capped at a literal row count (``LIMIT 10``, ``TOP 5``)."""
+ return bool(_SQL_LIMIT.search(text))
+
+
+def names_repo_root(name: str) -> bool:
+ """``repo_root`` / ``self._repo_path`` / ``projectDir``: the tree a repository is."""
+ ws = words(name)
+ return any(a in _ROOT_OWNERS and b in _ROOT_WORDS for a, b in pairwise(ws))
+
+
+__all__ = [
+ "has_sql_limit",
+ "is_config_model",
+ "is_config_name",
+ "is_config_sql",
+ "is_key_lookup",
+ "is_parent_key_filter",
+ "is_rare_filter",
+ "names_repo_root",
+ "sql_tables",
+ "words",
+]
diff --git a/packages/core/src/repowise/core/analysis/health/refactoring/performance_fix.py b/packages/core/src/repowise/core/analysis/health/refactoring/performance_fix.py
index 25355d2d5..e8a665147 100644
--- a/packages/core/src/repowise/core/analysis/health/refactoring/performance_fix.py
+++ b/packages/core/src/repowise/core/analysis/health/refactoring/performance_fix.py
@@ -53,6 +53,18 @@ _API_STEPS: dict[str, tuple[str | None, str]] = {
),
}
+# What a site's loop walks, as a step names it; an unknown source adds nothing.
+_ITERATES: dict[str, str] = {
+ "query": "rows a query returns",
+ "parameter": "a collection its caller passes in",
+ "caller_key": "the rows of one key",
+ "rare_subset": "the rows a filter keeps for a rare state",
+ "config_table": "a registry or configuration table",
+ "constant": "a fixed set",
+ "listing": "a directory listing",
+ "stream": "a stream as it arrives",
+}
+
@dataclass(frozen=True, slots=True)
class PerformancePlanPolicy:
@@ -125,11 +137,13 @@ def _site_steps(
if named and not api:
return [
{
- "action": named.format(name=edit["subject"]) if edit.get("subject") else per_site,
+ "action": (named.format(name=edit["subject"]) if edit.get("subject") else per_site)
+ + _walks(edit.get("loop_iterates")),
"symbol": edit.get("function_name"),
"file_path": edit.get("file_path"),
"line": edit.get("loop_line") or edit.get("line_start"),
**({"loop_line": edit["loop_line"]} if edit.get("loop_line") else {}),
+ **({"loop_iterates": what} if (what := edit.get("loop_iterates")) else {}),
"applicability": applicability,
}
for edit in _loop_edits(locations)
@@ -138,11 +152,13 @@ def _site_steps(
{
"action": per_site.format(
api=api, call=_call_phrase(location.get("call"), boundary)
- ),
+ )
+ + _walks(location.get("loop_iterates")),
"symbol": location.get("function_name"),
"file_path": location.get("file_path"),
"line": location.get("line_start"),
**({"loop_line": location["loop_line"]} if location.get("loop_line") else {}),
+ **({"loop_iterates": what} if (what := location.get("loop_iterates")) else {}),
"applicability": applicability,
}
for location in locations
@@ -175,6 +191,11 @@ def _call_phrase(call: object, boundary: str | None) -> str:
return " ".join(str(part) for part in (call, noun, "call") if part)
+def _walks(iterates: object) -> str:
+ phrase = _ITERATES.get(iterates) if isinstance(iterates, str) else None
+ return f" (the loop walks {phrase})" if phrase else ""
+
+
_EFFORT_BANDS = ((1, "S"), (3, "M"), (8, "L"))
@@ -239,6 +260,8 @@ def _site_locations(evidence: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
site["call"] = call
if "subject" not in site and item.get("subject"):
site["subject"] = item["subject"]
+ if "loop_iterates" not in site and item.get("loop_iterates"):
+ site["loop_iterates"] = item["loop_iterates"]
return list(sites.values())
diff --git a/tests/unit/health/test_perf_crossfn.py b/tests/unit/health/test_perf_crossfn.py
index eeff8bbf8..6ea76a3ce 100644
--- a/tests/unit/health/test_perf_crossfn.py
+++ b/tests/unit/health/test_perf_crossfn.py
@@ -538,8 +538,8 @@ def test_crossfn_hit_carries_the_owning_loops_magnitude(tmp_path):
count travels with the cross-function hit into the finding."""
source = _SCHEDULER.replace(
" for repo in repos:",
- " repos = self.session.execute(select(Repo)).scalars().all()\n"
- " for repo in repos:",
+ " orders = self.session.execute(select(Order)).scalars().all()\n"
+ " for repo in orders:",
)
walked, graph = _build(tmp_path, {"scheduler.py": source})
(hit,) = [h for hs in collect_crossfn_io_in_loop(walked, graph).values() for h in hs]
diff --git a/tests/unit/health/test_perf_loop_facts_python.py b/tests/unit/health/test_perf_loop_facts_python.py
index 32740f720..a3a2fd455 100644
--- a/tests/unit/health/test_perf_loop_facts_python.py
+++ b/tests/unit/health/test_perf_loop_facts_python.py
@@ -209,11 +209,12 @@ def test_magnitude_parameter_iterable_is_unknown():
"""Parameters are unknown on purpose — no reaching assignment to resolve."""
facts = _loop(
b"from sqlalchemy import select\n"
- b"async def f(session, repos):\n"
- b" for r in repos:\n"
+ b"async def f(session, items):\n"
+ b" for r in items:\n"
b" await session.execute(select(r))\n"
)
- assert facts is None # unknown magnitude + no batch/bound -> no LoopFacts at all
+ assert facts is not None and facts.magnitude == "unknown"
+ assert facts.iterates == "parameter"
# ---------------------------------------------------------------------------
@@ -275,7 +276,7 @@ def test_batch_none_when_loop_var_used_twice_in_chain():
b" await client.table('t').select('*')"
b".eq('repo_id', r.id).eq('name', r.name).execute()\n"
)
- assert facts is None
+ assert facts is None or facts.batch is None
def test_batch_supabase_limit_makes_not_equivalent():
@@ -306,7 +307,7 @@ def test_batch_second_sink_in_body_makes_not_equivalent():
b" session.execute(select(r))\n"
b" await client.table('t').select('*').eq('repo_id', r.id).execute()\n"
)
- supabase_hit = next(h for h in hits if h.loop is not None)
+ supabase_hit = next(h for h in hits if h.line == 5)
assert supabase_hit.loop.batch.equivalent is False
@@ -330,7 +331,7 @@ def test_batch_none_when_iterable_appended_in_body():
b" repos.append(r)\n"
b" await client.table('t').select('*').eq('repo_id', r.id).execute()\n"
)
- assert facts is None
+ assert facts is None or facts.batch is None
def test_batch_none_for_while_loop():
@@ -385,7 +386,7 @@ def test_concurrency_bound_none_when_loop_has_break():
b" async with self._sem:\n"
b" await session.execute(select(x))\n"
)
- assert facts is None
+ assert facts is None or facts.concurrency_bound is None
def test_concurrency_bound_none_when_loop_has_return():
@@ -398,7 +399,7 @@ def test_concurrency_bound_none_when_loop_has_return():
b" async with self._sem:\n"
b" await session.execute(select(x))\n"
)
- assert facts is None
+ assert facts is None or facts.concurrency_bound is None
def test_concurrency_bound_none_for_sync_with():
@@ -410,7 +411,7 @@ def test_concurrency_bound_none_for_sync_with():
b" with self._sem:\n"
b" await session.execute(select(x))\n"
)
- assert facts is None
+ assert facts is None or facts.concurrency_bound is None
def test_batch_none_when_the_key_is_computed_from_the_element():
diff --git a/tests/unit/health/test_perf_loop_sources.py b/tests/unit/health/test_perf_loop_sources.py
new file mode 100644
index 000000000..d04c89a72
--- /dev/null
+++ b/tests/unit/health/test_perf_loop_sources.py
@@ -0,0 +1,439 @@
+"""What a perf loop walks, the magnitude that proves, and whether its calls can be one batch.
+
+Through the real walker, as the other loop-fact tests do. Each positive rule is
+paired with the shape that must stay ``unknown`` or ``query``.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.analysis.health.complexity import walk_file
+from repowise.core.analysis.health.perf import loop_lexicon
+from repowise.core.analysis.health.perf.actionability import actionability, assess_fix
+from repowise.core.analysis.health.refactoring.performance_fix import fix_steps
+
+
+def _loop(path: str, language: str, src: str):
+ hits = [h for h in walk_file(path, language, src.encode()).perf_hits if h.kind == "io_in_loop"]
+ assert hits, "no io_in_loop hit"
+ return hits[0].loop
+
+
+def _py(src: str):
+ return _loop("f.py", "python", src)
+
+
+def _ts(src: str):
+ return _loop("f.ts", "typescript", src)
+
+
+_REGISTERED_REPOS = """
+from sqlalchemy import select
+async def check(session):
+ result = await session.execute(select(Repository))
+ repos = result.scalars().all()
+ for repo in repos:
+ await session.execute(select(Page).where(Page.repository_id == repo.id))
+"""
+# The same loop with a neutral variable name, so only the model can say "registry".
+_REGISTERED_ROWS = _REGISTERED_REPOS.replace("repos = ", "rows = ").replace("in repos", "in rows")
+
+
+# -- registry and configuration tables ------------------------------------------
+
+
+def test_a_loop_over_the_registered_repos_table_still_grows():
+ # Registered repos grow with use: the table's name says what is walked, not how many.
+ facts = _py(_REGISTERED_ROWS)
+ assert facts.iterates == "config_table"
+ assert facts.magnitude == "grows_with_data"
+
+
+def test_a_loop_over_another_table_is_a_query_that_grows():
+ src = _REGISTERED_REPOS.replace("Repository", "Order").replace("repos", "orders")
+ facts = _py(src)
+ assert facts.iterates == "query"
+ assert facts.magnitude == "grows_with_data"
+
+
+def test_a_workspace_config_repos_list_is_a_config_table():
+ facts = _py(
+ "from sqlalchemy import select\n"
+ "async def sync(session, ws_config):\n"
+ " for entry in ws_config.repos:\n"
+ " await session.execute(select(Repo).where(Repo.path == entry.path))\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("config_table", "unknown")
+
+
+def test_a_control_flow_graph_named_cfg_is_not_configuration():
+ assert not loop_lexicon.is_config_name("cfg.blocks")
+ assert loop_lexicon.is_config_name("models_cfg")
+ assert not loop_lexicon.is_config_name("_REGISTRY_RE.finditer", callee=True)
+ # A function returning repos may be scanning a disk for them.
+ assert not loop_lexicon.is_config_name("discover_repos", callee=True)
+
+
+# -- one key, a rare state ----------------------------------------------------------
+
+
+def test_rows_of_one_primary_key_are_bounded():
+ facts = _py(
+ "def f(conn, task_id):\n"
+ " rows = conn.execute('SELECT * FROM tasks WHERE id = ?', (task_id,)).fetchall()\n"
+ " for row in rows:\n"
+ " conn.execute('UPDATE runs SET x = 1 WHERE task = ?', (row[0],)).fetchone()\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("caller_key", "bounded")
+
+
+ "where",
+ ["parent_id = ?", "id = m.session_id", "id = ? UNION SELECT id FROM t"],
+)
+def test_a_foreign_key_or_a_correlated_id_is_not_one_key(where):
+ facts = _py(
+ "def f(conn, task_id):\n"
+ f" rows = conn.execute('SELECT * FROM tasks WHERE {where}', (task_id,)).fetchall()\n"
+ " for row in rows:\n"
+ " conn.execute('UPDATE runs SET x = 1 WHERE task = ?', (row[0],)).fetchone()\n"
+ )
+ assert facts.iterates != "caller_key"
+ assert facts.magnitude == "grows_with_data"
+
+
+def test_a_filter_for_expired_rows_is_a_rare_subset_that_still_grows():
+ facts = _py(
+ "def f(conn, now):\n"
+ " stale = conn.execute(\n"
+ " 'SELECT id FROM tasks WHERE status = ? AND claim_expires < ?', ('running', now)\n"
+ " ).fetchall()\n"
+ " for row in stale:\n"
+ " conn.execute('UPDATE tasks SET claim = NULL WHERE id = ?', (row[0],)).fetchone()\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("rare_subset", "grows_with_data")
+
+
+# -- parameters, constants, listings --------------------------------------------------
+
+
+def test_a_parameter_is_named_and_stays_unknown():
+ facts = _py(
+ "from sqlalchemy import select\n"
+ "async def f(session, items):\n"
+ " for item in items:\n"
+ " await session.execute(select(M).where(M.id == item))\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("parameter", "unknown")
+
+
+def test_a_chunked_walk_names_the_collection_it_chunks():
+ facts = _py(
+ "from sqlalchemy import select\n"
+ "_CHUNK = 500\n"
+ "async def f(session, ids):\n"
+ " for i in range(0, len(ids), _CHUNK):\n"
+ " await session.execute(select(M).where(M.id.in_(ids[i:i + _CHUNK])))\n"
+ )
+ assert facts.iterates == "parameter"
+
+
+def test_a_class_constant_bound_to_a_literal_is_a_constant():
+ src = (
+ "class Store:\n"
+ " _TABLES = ('a', 'b')\n"
+ " def wipe(self, conn):\n"
+ " for t in self._TABLES:\n"
+ " conn.execute('DELETE FROM x WHERE t = ?', (t,)).fetchone()\n"
+ )
+ assert (_py(src).iterates, _py(src).magnitude) == ("constant", "bounded")
+ # Bound to anything but a literal, the spelling proves nothing.
+ computed = src.replace("('a', 'b')", "load_tables()")
+ assert _py(computed) is None or _py(computed).magnitude == "unknown"
+
+
+def test_a_class_constant_listing_computed_entries_is_no_constant():
+ src = (
+ "class Store:\n"
+ " _ROOTS = [BASE / 'a', repo.path]\n"
+ " def wipe(self, conn):\n"
+ " for t in self._ROOTS:\n"
+ " conn.execute('DELETE FROM x WHERE t = ?', (t,)).fetchone()\n"
+ )
+ assert _py(src) is None or _py(src).iterates != "constant"
+
+
+def test_a_fixed_width_slice_keeps_what_the_loop_walks():
+ facts = _py(
+ "from sqlalchemy import select\n"
+ "async def f(session, ids):\n"
+ " for key in ids[:5]:\n"
+ " await session.execute(select(M).where(M.id == key))\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("parameter", "bounded")
+
+
+def test_ts_a_fixed_width_slice_keeps_what_the_loop_walks():
+ facts = _ts(
+ "async function f(ids: string[]) {\n"
+ " for (const id of ids.slice(0, 5)) {\n"
+ " await prisma.page.findMany({ where: { id } });\n"
+ " }\n"
+ "}\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("parameter", "bounded")
+
+
+def test_a_recursive_listing_of_the_repository_root_grows():
+ facts = _py(
+ "import os\n"
+ "from sqlalchemy import select\n"
+ "async def f(session, repo_root):\n"
+ " for root_, dirs, files in os.walk(repo_root):\n"
+ " await session.execute(select(root_))\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("listing", "grows_with_data")
+
+
+def test_a_flat_listing_is_a_listing_of_unknown_size():
+ facts = _py(
+ "from sqlalchemy import select\n"
+ "async def f(session, repo_root):\n"
+ " for p in repo_root.iterdir():\n"
+ " await session.execute(select(p))\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("listing", "unknown")
+
+
+# -- not batchable ---------------------------------------------------------------------
+
+
+def test_a_loop_that_returns_the_first_hit_is_not_batchable():
+ facts = _py(
+ "from sqlalchemy import select\n"
+ "async def find(session, keys):\n"
+ " for key in keys:\n"
+ " row = await session.get(Model, key)\n"
+ " if row is not None:\n"
+ " return row\n"
+ " return None\n"
+ )
+ assert facts.not_batchable == "first_match"
+
+
+def test_a_fallback_that_returns_the_first_call_that_works_is_not_batchable():
+ facts = _py(
+ "import requests\n"
+ "def fetch(urls):\n"
+ " for url in urls:\n"
+ " try:\n"
+ " return requests.get(url)\n"
+ " except OSError:\n"
+ " continue\n"
+ )
+ assert facts.not_batchable == "first_match"
+
+
+def test_an_exit_on_a_local_derived_from_the_result_is_no_first_match():
+ # ``ok`` may test anything about the row; only a test of the result itself
+ # says the call answered.
+ facts = _py(
+ "from sqlalchemy import select\n"
+ "async def find(session, keys):\n"
+ " for key in keys:\n"
+ " row = await session.get(Model, key)\n"
+ " ok = check(row)\n"
+ " if ok:\n"
+ " break\n"
+ )
+ assert facts.not_batchable is None
+
+
+def test_an_exit_under_an_outer_test_of_the_result_is_no_first_match():
+ facts = _py(
+ "from sqlalchemy import select\n"
+ "async def find(session, keys):\n"
+ " for key in keys:\n"
+ " row = await session.get(Model, key)\n"
+ " if row is not None:\n"
+ " if key.last:\n"
+ " break\n"
+ )
+ assert facts.not_batchable is None
+
+
+def test_a_cap_on_what_the_loop_collected_is_no_first_match():
+ facts = _py(
+ "import requests\n"
+ "def fetch(urls, out):\n"
+ " for url in urls:\n"
+ " out.append(requests.get(url))\n"
+ " if len(out) >= 5:\n"
+ " break\n"
+ )
+ assert facts.not_batchable is None
+
+
+def test_an_exit_on_the_element_is_no_first_match():
+ facts = _py(
+ "import requests\n"
+ "def fetch(urls):\n"
+ " for url in urls:\n"
+ " if url.done:\n"
+ " break\n"
+ " requests.get(url)\n"
+ )
+ assert facts.not_batchable is None
+
+
+def test_a_cursor_fed_by_the_previous_page_is_a_turn_loop():
+ facts = _py(
+ "import requests\n"
+ "def pages(url):\n"
+ " cursor = None\n"
+ " while True:\n"
+ " page = requests.get(url, params={'cursor': cursor}).json()\n"
+ " cursor = page.get('next')\n"
+ )
+ assert facts.not_batchable == "turn_loop"
+
+
+def test_a_conversation_appended_to_each_turn_is_a_turn_loop():
+ facts = _py(
+ "import requests\n"
+ "def chat(url, messages, turns):\n"
+ " for _ in turns:\n"
+ " reply = requests.post(url, json=messages)\n"
+ " messages.append(reply)\n"
+ )
+ assert facts.not_batchable == "turn_loop"
+
+
+def test_grouping_results_by_the_element_is_no_turn_loop():
+ facts = _py(
+ "import requests\n"
+ "def f(rows, by_file):\n"
+ " for row in rows:\n"
+ " entry = requests.get(row.url)\n"
+ " by_file.setdefault(row.path, []).append(entry)\n"
+ )
+ assert facts.not_batchable is None
+
+
+def test_independent_calls_are_not_marked():
+ facts = _py(
+ "import requests\n"
+ "def total(urls):\n"
+ " n = 0\n"
+ " for url in urls:\n"
+ " r = requests.get(url)\n"
+ " n += len(r.content)\n"
+ " return n\n"
+ )
+ assert facts.not_batchable is None
+
+
+# -- TypeScript --------------------------------------------------------------------------
+
+
+def test_ts_a_counted_loop_to_a_literal_or_a_module_constant_is_bounded():
+ for bound, header in (("5", ""), ("MAX_TRIES", "const MAX_TRIES = 5;\n")):
+ facts = _ts(
+ header + "async function f(u: string) {\n"
+ f" for (let i = 0; i < {bound}; i++) {{\n"
+ " await fetch(u);\n"
+ " }\n"
+ "}\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("constant", "bounded")
+
+
+def test_ts_a_counted_loop_over_a_read_grows():
+ facts = _ts(
+ "async function f() {\n"
+ " const rows = await prisma.order.findMany();\n"
+ " for (let i = 0; i < rows.length; i++) {\n"
+ " await prisma.item.findMany({ where: { orderId: rows[i].id } });\n"
+ " }\n"
+ "}\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("query", "grows_with_data")
+
+
+def test_ts_a_registry_table_read_still_grows():
+ facts = _ts(
+ "async function f() {\n"
+ " const rows = await prisma.repository.findMany();\n"
+ " for (const r of rows) {\n"
+ " await prisma.page.findMany({ where: { repoId: r.id } });\n"
+ " }\n"
+ "}\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("config_table", "grows_with_data")
+
+
+def test_ts_walking_the_repository_tree_grows():
+ facts = _ts(
+ "import fs from 'fs';\n"
+ "function f(repoRoot: string) {\n"
+ " for (const file of walkFiles(path.join(repoRoot, 'src'))) {\n"
+ " fs.readFileSync(file);\n"
+ " }\n"
+ "}\n"
+ )
+ assert (facts.iterates, facts.magnitude) == ("listing", "grows_with_data")
+
+
+def test_ts_first_match_and_parameter():
+ facts = _ts(
+ "async function f(urls: string[]) {\n"
+ " for (const u of urls) {\n"
+ " const res = await fetch(u);\n"
+ " if (res.ok) return res;\n"
+ " }\n"
+ "}\n"
+ )
+ assert facts.iterates == "parameter"
+ assert facts.not_batchable == "first_match"
+
+
+# -- what readers see -------------------------------------------------------------------
+
+
+def _db_details(**extra):
+ return [{"loop_magnitude": "grows_with_data", **extra}]
+
+
+def test_a_not_batchable_loop_gets_no_batch_advice():
+ assessment = assess_fix(
+ "io_in_loop", ("io_in_loop",), "db", _db_details(loop_not_batchable="first_match"),
+ cross_function=False,
+ )
+ assert assessment.fix is None and assessment.refusal == "loop_not_batchable"
+ # A fact about the code, not a missing proof: nothing to investigate.
+ assert actionability(assessment, "high").state == "expected"
+
+
+def test_one_batchable_member_keeps_the_strategy():
+ details = [*_db_details(loop_not_batchable="first_match"), *_db_details()]
+ assessment = assess_fix("io_in_loop", ("io_in_loop",), "db", details, cross_function=False)
+ assert assessment.fix is not None
+
+
+def test_a_plan_step_says_what_the_loop_walks():
+ (step,) = fix_steps(
+ "batch_or_prefetch_io",
+ "advisory",
+ None,
+ None,
+ [{"file_path": "a.py", "function_name": "f", "line_start": 3, "call": "session.get",
+ "loop_iterates": "query"}],
+ )
+ assert step["action"].endswith("(the loop walks rows a query returns)")
+ assert step["loop_iterates"] == "query"
+ (plain,) = fix_steps(
+ "batch_or_prefetch_io", "advisory", None, None,
+ [{"file_path": "a.py", "function_name": "f", "line_start": 3, "loop_iterates": "unknown"}],
+ )
+ assert "loop walks" not in plain["action"]
diff --git a/tests/unit/health/test_perf_loop_sources_dialects.py b/tests/unit/health/test_perf_loop_sources_dialects.py
new file mode 100644
index 000000000..18482bf98
--- /dev/null
+++ b/tests/unit/health/test_perf_loop_sources_dialects.py
@@ -0,0 +1,410 @@
+"""What a perf loop walks, and the magnitude that proves, in Go, Java, C# and Ruby.
+
+Through the real walker. Each shape that grows or is bounded is paired with one
+the syntax does not settle, which must stay ``unknown``: no reader takes
+``unknown`` as bounded.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.analysis.health.complexity import walk_file
+
+
+def _loops(path: str, language: str, src: str) -> dict[int, tuple[str, str]]:
+ """``{hit line: (magnitude, iterates)}`` for every io_in_loop hit."""
+ out: dict[int, tuple[str, str]] = {}
+ for hit in walk_file(path, language, src.encode()).perf_hits:
+ if hit.kind == "io_in_loop":
+ facts = hit.loop
+ out[hit.line] = (facts.magnitude, facts.iterates) if facts else ("unknown", "unknown")
+ return out
+
+
+_GO = """package m
+import "database/sql"
+func f(db *sql.DB, ids []int) {
+ rows, _ := db.Query("SELECT id FROM orders")
+ for rows.Next() { db.Exec("UPDATE x SET y = 1") }
+ for _, id := range ids { db.Exec("DELETE FROM t WHERE id = ?", id) }
+ for _, id := range ids[:5] { db.Exec("DELETE FROM t WHERE id = ?", id) }
+ for i := 0; i < len(ids); i++ { db.Exec("DELETE FROM t WHERE id = ?", ids[i]) }
+ var users []User
+ db.Find(&users)
+ for _, u := range users { db.Exec("x", u) }
+ for _, s := range []string{"a", "b"} { db.Exec("x", s) }
+ one, _ := db.Query("SELECT * FROM t WHERE id = ?", 3)
+ for one.Next() { db.Exec("x") }
+ top, _ := db.Query("SELECT * FROM t LIMIT 10")
+ for top.Next() { db.Exec("x") }
+ for _, r := range repos { db.Exec("x", r) }
+}
+"""
+
+
+ ("line", "expected"),
+ [
+ (5, ("grows_with_data", "query")), # a sql.Rows cursor over a table
+ (6, ("unknown", "parameter")),
+ (7, ("bounded", "parameter")), # a fixed slice keeps what it walks
+ (8, ("unknown", "parameter")),
+ (11, ("grows_with_data", "query")), # GORM fills ``&users``
+ (12, ("bounded", "constant")),
+ (14, ("bounded", "caller_key")),
+ (16, ("unknown", "query")), # capped in its SQL
+ (17, ("unknown", "config_table")), # a registry's name bounds nothing
+ ],
+)
+def test_go_loop_sources(line, expected):
+ assert _loops("f.go", "go", _GO)[line] == expected
+
+
+_JAVA = """import java.sql.*;
+class A {
+ void f(List<Long> ids, UserRepository userRepository) {
+ List<User> us = userRepository.findAll();
+ for (User u : us) { userRepository.findById(u.getId()); }
+ for (Long id : ids) { userRepository.findById(id); }
+ for (int i = 0; i < us.size(); i++) { userRepository.findById(ids.get(i)); }
+ ResultSet rs = stmt.executeQuery("SELECT * FROM orders");
+ while (rs.next()) { userRepository.findById(rs.getLong(1)); }
+ for (String s : List.of("a", "b")) { userRepository.findByName(s); }
+ for (Role r : Role.values()) { userRepository.findByRole(r); }
+ for (User u : userRepository.findAll().stream().limit(5).toList()) { userRepository.findById(u.getId()); }
+ for (User u : userRepository.findAll().stream().filter(x -> x.ok).toList()) { userRepository.findById(u.getId()); }
+ for (Long id : Arrays.asList(ids)) { userRepository.findById(id); }
+ }
+}
+"""
+
+
+ ("line", "expected"),
+ [
+ (5, ("grows_with_data", "query")),
+ (6, ("unknown", "parameter")),
+ (7, ("grows_with_data", "query")), # ``i < us.size()`` walks ``us``
+ (9, ("grows_with_data", "query")), # a JDBC ResultSet cursor
+ (10, ("bounded", "constant")),
+ (11, ("bounded", "constant")), # an enum's constants
+ (12, ("bounded", "query")),
+ (13, ("grows_with_data", "query")),
+ (14, ("unknown", "parameter")), # ``asList`` of one array walks it
+ ],
+)
+def test_java_loop_sources(line, expected):
+ assert _loops("A.java", "java", _JAVA)[line] == expected
+
+
+_CS = """using Microsoft.EntityFrameworkCore;
+class A {
+ async Task F(List<int> ids) {
+ var us = await _context.Users.Where(u => u.Active).ToListAsync();
+ foreach (var u in us) { await _context.Orders.FirstOrDefaultAsync(o => o.UserId == u.Id); }
+ foreach (var id in ids) { await _context.Orders.FirstOrDefaultAsync(o => o.UserId == id); }
+ foreach (var id in ids.Where(x => x > 0).ToList()) { await _context.Orders.FirstOrDefaultAsync(o => o.UserId == id); }
+ foreach (var u in _context.Users) { await _context.Orders.FirstOrDefaultAsync(o => o.UserId == u.Id); }
+ foreach (var s in new[] { "a", "b" }) { await _context.Orders.FirstOrDefaultAsync(o => o.Name == s); }
+ for (int i = 0; i < ids.Count; i++) { await _context.Orders.FirstOrDefaultAsync(o => o.UserId == ids[i]); }
+ using var reader = await cmd.ExecuteReaderAsync();
+ while (await reader.ReadAsync()) { await _context.Orders.FirstOrDefaultAsync(o => o.UserId == 1); }
+ foreach (var x in ids.Take(5)) { await _context.Orders.FirstOrDefaultAsync(o => o.UserId == x); }
+ foreach (var y in [1, ..ids]) { await _context.Orders.FirstOrDefaultAsync(o => o.UserId == y); }
+ }
+}
+"""
+
+
+ ("line", "expected"),
+ [
+ (5, ("grows_with_data", "query")),
+ (6, ("unknown", "parameter")),
+ # Sync LINQ on a list the caller passed is not a database read.
+ (7, ("unknown", "parameter")),
+ (8, ("grows_with_data", "query")), # walking a DbSet reads the table
+ (9, ("bounded", "constant")),
+ (10, ("unknown", "parameter")),
+ (12, ("grows_with_data", "query")), # a data reader cursor
+ (13, ("bounded", "parameter")),
+ (14, ("unknown", "unknown")), # a spread is as long as what it spreads
+ ],
+)
+def test_csharp_loop_sources(line, expected):
+ assert _loops("A.cs", "csharp", _CS)[line] == expected
+
+
+_RUBY = """require "active_record"
+def f(ids)
+ users = User.where(active: true)
+ users.each do |u|
+ Order.where(user_id: u.id).to_a
+ end
+ User.find_each { |u| Order.where(user_id: u.id).to_a }
+ for x in ids
+ Order.where(user_id: x).to_a
+ end
+ %w[a b].each { |s| Order.where(name: s).to_a }
+ ids.first(5).each_with_index { |x, i| Order.where(user_id: x).to_a }
+ ids.size.times { |i| Order.where(user_id: ids[i]).to_a }
+ User.all.each { |u| Order.where(user_id: u.id).to_a }
+ Dir.glob("*.rb").each { |p| Order.where(path: p).to_a }
+ @posts.each { |p| Order.where(post_id: p.id).to_a }
+ User.limit(10).each { |u| Order.where(user_id: u.id).to_a }
+end
+"""
+
+
+ ("line", "expected"),
+ [
+ (5, ("grows_with_data", "query")), # a relation on a model, one hop back
+ (7, ("grows_with_data", "query")), # find_each walks the table in batches
+ (9, ("unknown", "parameter")),
+ (11, ("bounded", "constant")),
+ (12, ("bounded", "parameter")),
+ (13, ("unknown", "parameter")), # ``ids.size.times`` walks ``ids``
+ (14, ("grows_with_data", "query")),
+ (15, ("unknown", "listing")), # a flat glob of unknown size
+ (16, ("unknown", "unknown")),
+ (17, ("unknown", "query")), # capped in the relation
+ ],
+)
+def test_ruby_loop_sources(line, expected):
+ assert _loops("a.rb", "ruby", _RUBY)[line] == expected
+
+
+def test_a_ruby_block_loop_never_claims_its_key_is_unused():
+ # Facts on block loops say what they walk; they leave batch advice as it was.
+ hits = walk_file(
+ "a.rb",
+ "ruby",
+ b'require "active_record"\n'
+ b"def f(ids)\n"
+ b" ids.each { |id| Order.find_by(id: 1) }\n"
+ b"end\n",
+ ).perf_hits
+ facts = [h.loop for h in hits if h.kind == "io_in_loop"]
+ assert facts and all(f is None or not f.key_unused for f in facts)
+
+
+def test_an_empty_literal_filled_before_the_loop_is_no_constant():
+ cs = _loops(
+ "A.cs",
+ "csharp",
+ "using Microsoft.EntityFrameworkCore;\n"
+ "class A {\n"
+ " async Task F(List<int> ids) {\n"
+ " List<int> made = [];\n"
+ " foreach (var id in ids) { made.Add(id); }\n"
+ " foreach (var m in made) { await _context.Orders.FirstOrDefaultAsync(o => o.Id == m); }\n"
+ " }\n"
+ "}\n",
+ )
+ assert cs[6] == ("unknown", "unknown")
+ rb = _loops(
+ "a.rb",
+ "ruby",
+ 'require "active_record"\n'
+ "def f(xs)\n"
+ " seen = ['a']\n"
+ " xs.each { |x| seen << x }\n"
+ " seen.each { |s| Order.find_by(name: s) }\n"
+ "end\n",
+ )
+ assert rb[5] == ("unknown", "unknown")
+
+
+def test_a_counter_stepped_after_its_literal_is_no_constant():
+ cs = _loops(
+ "A.cs",
+ "csharp",
+ "using Microsoft.EntityFrameworkCore;\n"
+ "class A {\n"
+ " async Task F(int total) {\n"
+ " int rows = 1;\n"
+ " while (total / rows >= rows) { rows++; }\n"
+ " for (int r = 0; r < rows; r++) { await _context.Orders.FirstOrDefaultAsync(o => o.Id == r); }\n"
+ " }\n"
+ "}\n",
+ )
+ assert cs[6] == ("unknown", "unknown")
+
+
+def test_go_a_slice_written_by_create_is_still_the_literal():
+ go = _loops(
+ "f.go",
+ "go",
+ 'package m\nimport "database/sql"\n'
+ "func f(db *sql.DB) {\n"
+ ' menus := []Menu{{Name: "a"}, {Name: "b"}}\n'
+ " db.Create(&menus)\n"
+ ' for _, m := range menus { db.Exec("x", m) }\n'
+ "}\n",
+ )
+ assert go[6] == ("bounded", "constant")
+
+
+def test_go_an_empty_slice_literal_is_no_constant():
+ go = _loops(
+ "f.go",
+ "go",
+ 'package m\nimport "database/sql"\n'
+ "func f(db *sql.DB) {\n"
+ " xs := []string{}\n"
+ ' for _, x := range xs { db.Exec("x", x) }\n'
+ "}\n",
+ )
+ assert go[5] == ("unknown", "unknown")
+
+
+def test_java_a_recursive_walk_of_the_repository_root_grows():
+ java = _loops(
+ "A.java",
+ "java",
+ "import java.nio.file.*;\n"
+ "class A { void f(Path repoRoot, Path dir, UserRepository userRepository) {\n"
+ " for (Path p : Files.walk(repoRoot).toList()) { userRepository.findById(1L); }\n"
+ " for (Path p : Files.list(dir).toList()) { userRepository.findById(1L); } } }\n",
+ )
+ assert java[3] == ("grows_with_data", "listing")
+ assert java[4] == ("unknown", "listing")
+
+
+def test_the_children_of_one_parent_are_a_query_of_unknown_size():
+ go = _loops(
+ "f.go",
+ "go",
+ 'package m\nimport "database/sql"\n'
+ "func f(db *sql.DB, id int) {\n"
+ " var menus []Menu\n"
+ ' db.Where("authority_id = ?", id).Find(&menus)\n'
+ ' for _, m := range menus { db.Exec("x", m) }\n'
+ "}\n",
+ )
+ assert go[6] == ("unknown", "query")
+ rb = _loops(
+ "a.rb",
+ "ruby",
+ 'require "active_record"\n'
+ "def f(user)\n"
+ " Post.where(user_id: user.id).each { |p| Comment.find_by(post_id: p.id) }\n"
+ " User.active.joins(:posts).limit(30).each { |u| Comment.find_by(user_id: u.id) }\n"
+ "end\n",
+ )
+ assert rb[3] == ("unknown", "query")
+ assert rb[4][0] == "unknown" # a read's own cap, on a scope we cannot see into
+
+
+def test_a_binding_under_a_branch_leaves_the_loop_unknown():
+ # ``items`` is a read or, on one path, a literal: neither size is sure.
+ go = _loops(
+ "f.go",
+ "go",
+ 'package m\nimport "database/sql"\n'
+ "func f(db *sql.DB) {\n"
+ " var items []Item\n"
+ " db.Find(&items)\n"
+ " if len(items) == 0 {\n"
+ ' items = []Item{{Name: "a"}}\n'
+ " }\n"
+ ' for _, it := range items { db.Exec("x", it) }\n'
+ " var only []Item\n"
+ " if ok {\n"
+ ' only = []Item{{Name: "a"}}\n'
+ " }\n"
+ ' for _, it := range only { db.Exec("x", it) }\n'
+ "}\n",
+ )
+ assert go[9][0] == "unknown"
+ assert go[14][0] == "unknown"
+ java = _loops(
+ "A.java",
+ "java",
+ "class A { void f(boolean ok, UserRepository userRepository) {\n"
+ " List<String> names = List.of(\"a\", \"b\");\n"
+ " if (ok) { names = userRepository.findAllNames(); }\n"
+ " for (String n : names) { userRepository.findByName(n); } } }\n",
+ )
+ assert java[4][0] == "unknown"
+
+
+def test_a_read_in_an_if_initializer_still_reaches_the_loop():
+ go = _loops(
+ "f.go",
+ "go",
+ 'package m\nimport "database/sql"\n'
+ "func f(db *sql.DB) error {\n"
+ " var users []User\n"
+ " if err := db.Find(&users).Error; err != nil {\n"
+ " return err\n"
+ " }\n"
+ ' for _, u := range users { db.Exec("x", u) }\n'
+ " return nil\n"
+ "}\n",
+ )
+ assert go[8] == ("grows_with_data", "query")
+
+
+ ("path", "language", "src", "line"),
+ [
+ (
+ "a.rb",
+ "ruby",
+ 'require "active_record"\ndef f(ids, n)\n'
+ " ids.first(n).each { |x| Order.find_by(user: x) }\nend\n",
+ 3,
+ ),
+ (
+ "A.cs",
+ "csharp",
+ "using Microsoft.EntityFrameworkCore;\nclass A {\n"
+ " async Task F(List<int> ids, int n) {\n"
+ " foreach (var x in ids.Take(n)) { await _context.Orders.FirstOrDefaultAsync(o => o.UserId == x); }\n"
+ " }\n}\n",
+ 4,
+ ),
+ (
+ "A.java",
+ "java",
+ "class A { void f(List<Long> ids, int max, UserRepository userRepository) {\n"
+ " for (Long id : ids.stream().limit(max).toList()) { userRepository.findById(id); } } }\n",
+ 2,
+ ),
+ ],
+)
+def test_a_cap_that_is_not_a_literal_bounds_nothing(path, language, src, line):
+ assert _loops(path, language, src)[line][0] == "unknown"
+
+
+def test_returning_the_error_of_a_call_is_no_first_match():
+ hits = walk_file(
+ "f.go",
+ "go",
+ b'package m\nimport "database/sql"\n'
+ b"func f(db *sql.DB, ids []int) error {\n"
+ b" for _, id := range ids {\n"
+ b' if _, err := db.Exec("DELETE FROM t WHERE id = ?", id); err != nil {\n'
+ b" return err\n"
+ b" }\n"
+ b" }\n"
+ b" return nil\n"
+ b"}\n",
+ ).perf_hits
+ go = [h.loop for h in hits if h.kind == "io_in_loop"]
+ assert go and all(f is None or f.not_batchable is None for f in go)
+ hits = walk_file(
+ "f.py",
+ "python",
+ b"import requests\n"
+ b"def push(urls):\n"
+ b" for url in urls:\n"
+ b" resp, err = send(requests.post(url))\n"
+ b" if err:\n"
+ b" return err\n",
+ ).perf_hits
+ py = [h.loop for h in hits if h.kind == "io_in_loop"]
+ assert py and all(f is None or f.not_batchable is None for f in py)