Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(perf): prove fewer batch plans, and stop reading a plugin registry as a query (#2521)

Raghav Chamadiya69b167e25bfd
diff --git a/packages/core/src/repowise/core/analysis/health/engine.py b/packages/core/src/repowise/core/analysis/health/engine.py
index 093dfa3c5..daa51d080 100644
--- a/packages/core/src/repowise/core/analysis/health/engine.py
+++ b/packages/core/src/repowise/core/analysis/health/engine.py
@@ -111,6 +111,10 @@ log = structlog.get_logger(__name__)
# and a declining call's arguments are not scanned, so an assertion passed as an
# argument still does not stand in for the header's oracle.
#
+# v26: ``.all()`` on an imported or module-global name (a plugin registry) is not a
+# db sink, a SQL string saying ``LIMIT n`` caps a read, and a batched read is never
+# proven for ``session.get`` or beside a call made for its effect; a v25 store differs.
+#
# v25: repetition with nothing to change (a filesystem or subprocess boundary, or
# a loop already walking chunks) is stored as ``expected``; a v24 store says investigate.
#
@@ -253,7 +257,7 @@ log = structlog.get_logger(__name__)
# forms. Files that were counted untested and are not become tested, which
# moves untested-hotspot findings and the scores that carry them, on every
# language with a prefix or spec convention rather than Ruby alone.
-HEALTH_ANALYZER_VERSION = 25
+HEALTH_ANALYZER_VERSION = 26
def walked_functions(
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
index df2964b47..41c7f11ad 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
@@ -8,6 +8,7 @@ zero Python behavior — the defect golden + perf suite lock that.
from __future__ import annotations
+import builtins
import re
from typing import TYPE_CHECKING, ClassVar
@@ -171,6 +172,9 @@ _PY_RESULT_PROJECTIONS: frozenset[str] = frozenset(
{"all", "scalars", "fetchall", "json", "data", "rows"}
)
_PY_BOUND_NAME_RE = re.compile(r"(?i)retr|attempt")
+# Raw SQL capped in its own text: ``sa.text("... LIMIT 1000")``.
+_PY_SQL_LIMIT_RE = re.compile(rb"(?is)\bselect\b.*\blimit\s+\d+\b")
+_PY_SQL_COMMENT_RE = re.compile(rb"--[^\n]*|/\*.*?\*/", re.S)
# A per-key call that limits or orders its rows is not the read one IN query makes.
_PY_PER_KEY_LIMITS: frozenset[str] = frozenset(
{
@@ -182,6 +186,7 @@ _PY_ONE_ROW: frozenset[str] = _PY_PER_KEY_LIMITS | {"scalar"}
_PY_READ_CAPS: frozenset[str] = _PY_ONE_ROW - {"order", "order_by", "offset"}
_PY_WRITES: frozenset[str] = frozenset({"update", "insert", "upsert", "values"})
_PY_SCOPES: frozenset[str] = frozenset({"function_definition", "lambda", "class_definition"})
+_PY_BUILTINS: frozenset[str] = frozenset(dir(builtins))
_PY_SEM_NAME_RE = re.compile(r"(?i)(sem|semaphore|limiter|limit)$")
# A model class is CapWords; an ALL_CAPS constant (``API_URL``) is not.
_PY_MODEL_NAME_RE = re.compile(r"^[A-Z]\w*[a-z]\w*$")
@@ -293,6 +298,8 @@ class PythonPerfDialect(BasePerfDialect):
kind = super().call_sink_kind(
call, awaited=awaited, io_names=io_names, has_db_import=has_db_import
)
+ if kind == "db" and self._reads_a_registry(call, io_names):
+ return None
orm_get = self._orm_get(call) if kind is None else None
if orm_get is not None:
# ``get`` alone is ``dict.get`` / ``requests.get``; the model-class shape
@@ -301,6 +308,28 @@ class PythonPerfDialect(BasePerfDialect):
return "db" if db and not self._key_used_earlier(call, orm_get[1]) else None
return kind
+ def _reads_a_registry(self, call: Node, io_names: dict[str, str]) -> bool:
+ """``plugins.all()`` on an imported or module-global name reads a registry, not a
+ query result: a result is bound in the function that reads it."""
+ if self.callee_method_name(call) not in PY_DB_AMBIGUOUS:
+ return False
+ fn = call.child_by_field_name("function")
+ receiver = fn.child_by_field_name("object") if fn is not None else None
+ if receiver is None or receiver.type != "identifier":
+ return False
+ name = receiver.text or b""
+ return io_names.get(name.decode()) != "db" and not self._bound_locally(call, name)
+
+ def _bound_locally(self, call: Node, name: bytes) -> bool:
+ """*name* is assigned before *call* in its function, or is a parameter of it."""
+ if self._reaching_rhs(call, name) is not None:
+ return True
+ scope = call.parent
+ while scope is not None and scope.type != "function_definition":
+ scope = scope.parent
+ params = scope.child_by_field_name("parameters") if scope is not None else None
+ return params is not None and name.decode() in self._param_names(params)
+
def _key_used_earlier(self, call: Node, key: Node) -> bool:
"""An earlier call in this loop body gave the same session the same key, so the
row may already sit in the session and ``get`` answers from memory."""
@@ -814,12 +843,18 @@ class PythonPerfDialect(BasePerfDialect):
return None
def _caps_read(self, node: Node, loop: Node) -> bool:
- """``.limit(n)`` and peers, or ``.in_()`` over one chunk of keys
- (``xs[i:i + N]``, or the element of a loop that walks ``xs`` in chunks)."""
+ """``.limit(n)`` and peers, a SQL string saying ``LIMIT n``, or ``.in_()`` over
+ one chunk of keys."""
+ if node.type == "string":
+ return bool(_PY_SQL_LIMIT_RE.search(_PY_SQL_COMMENT_RE.sub(b"", node.text or b"")))
method = self.callee_method_name(node) if node.type == "call" else None
if method in _PY_READ_CAPS:
return True
- args = node.child_by_field_name("arguments") if method == "in_" else None
+ return method == "in_" and self._in_one_chunk(node, loop)
+
+ def _in_one_chunk(self, node: Node, loop: Node) -> bool:
+ """``.in_(xs[i:i + N])``, or ``.in_(chunk)`` where a loop walks ``xs`` in chunks."""
+ args = node.child_by_field_name("arguments")
keys = next((c for c in args.children if c.is_named), None) if args else None
if keys is None:
return False
@@ -920,19 +955,49 @@ class PythonPerfDialect(BasePerfDialect):
kept = self._unconditional(sink, body)
else:
kept = not self._builds_in_order(body)
- orm_get = self._orm_get(sink)
- if orm_get is not None and orm_get[1] == refs[0]:
- # The whole element as the key may be a composite (tuple) key, which
- # ``primary_key[0].in_`` does not match; only ``r.id`` names one column.
+ if self._orm_get(sink) is not None:
+ # ``get`` answers from the session's identity map when the row is already
+ # loaded, which no syntax shows: held out, half of these sites made no
+ # round trip. The bulk form is named, never proven.
kept = False
equivalent = (
kept
and not methods & _PY_PER_KEY_LIMITS
and not self._limited_downstream(sink, body)
and self._only_io_in_body(sink, body, probe)
+ and not self._calls_for_effect(sink, body)
)
return BatchForm(call, equivalent)
+ def _calls_for_effect(self, sink: Node, body: Node) -> bool:
+ """Another statement calls something this function did not build as a list, set
+ or dict (``run_task(doc)``, ``session.add(r)``, ``t = task.delay(d)``), which may
+ write what the batched read would read before it ran."""
+ for node in self._walk(body, prune=_PY_SCOPES):
+ if node.type == "expression_statement":
+ call, assigned = node.named_children[0], False
+ elif node.type == "assignment":
+ call, assigned = node.child_by_field_name("right"), True
+ else:
+ continue
+ if call is not None and call.type == "await":
+ call = next((c for c in call.children if c.is_named), None)
+ if call is None or call.type != "call" or self._within(call, sink):
+ continue
+ if self._has_effect(call, assigned=assigned):
+ return True
+ return False
+
+ def _has_effect(self, call: Node, *, assigned: bool) -> bool:
+ """Not ``seen.add(x)`` on a local collection, and, when its result is kept, not a
+ builtin (``str(r.id)``) or a method on a local (``res.first()``)."""
+ if not self.callee_is_attribute(call):
+ return not (assigned and self.callee_method_name(call) in _PY_BUILTINS)
+ rhs = self._reaching_rhs(call, (self.callee_root_name(call) or "").encode())
+ if rhs is None:
+ return True
+ return not (assigned or self._rhs_is_list(rhs) or self._rhs_is_nonlist_container(rhs))
+
def _limited_downstream(self, sink: Node, body: Node) -> bool:
"""The result is cut to one row after the call (``(await q).first()``,
``res = await q`` then ``res.scalar_one()``), which one IN query does not do."""
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py
index 23eb95efd..4e83cf66a 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py
@@ -476,12 +476,11 @@ class TsJsPerfDialect(BasePerfDialect):
return self._magnitude_of_expr(self._reaching_assignment(loop, right.text), probe)
return magnitude
- def _limiter_of(self, closure: Node) -> str | None:
- """The limiter an awaited call runs *closure* through (``await limit(() => f(x))``).
+ def _awaited_wrapper_call(self, closure: Node) -> Node | None:
+ """The awaited call *closure* is passed to (``await limit(() => f(x))``).
Awaited, the closure finishes before the loop moves on, so it is the loop
- body. Stored, returned or not awaited, it runs later and is not. A curried
- ``pLimit(2)(...)`` builds a fresh limiter per call, which bounds nothing.
+ body. Stored, returned or not awaited, it runs later and is not.
"""
args = closure.parent
if closure.type != "arrow_function" or args is None or args.type != "arguments":
@@ -492,9 +491,18 @@ class TsJsPerfDialect(BasePerfDialect):
fn = call.child_by_field_name("function")
if fn is None or fn.type not in ("identifier", "member_expression"):
return None
+ return call
+
+ def _limiter_of(self, closure: Node) -> str | None:
+ """The limiter *closure* runs through. A curried ``pLimit(2)(...)`` builds a
+ fresh limiter per call, which bounds nothing, so the callee must be named one."""
+ call = self._awaited_wrapper_call(closure)
+ if call is None:
+ return None
name = self.callee_method_name(call) or ""
if not (_LIMITER_NAME_RE.search(name) or name in _LIMITER_METHODS):
return None
+ fn = call.child_by_field_name("function")
return (fn.text or b"").decode() or None
def runs_in_place(self, closure: Node) -> bool:
diff --git a/tests/unit/health/test_model_version_coupling.py b/tests/unit/health/test_model_version_coupling.py
index 8a0ce256b..da6a3d0c2 100644
--- a/tests/unit/health/test_model_version_coupling.py
+++ b/tests/unit/health/test_model_version_coupling.py
@@ -22,7 +22,7 @@ from repowise.core.analysis.health.refactoring.identity import (
# The three stamps as they stand together. Moving either identity model means
# moving the analyzer with it, and updating this tuple in the same commit.
-_STAMPS = (25, 2, 2)
+_STAMPS = (26, 2, 2)
def test_the_identity_models_and_the_analyzer_stamp_are_pinned_together() -> None:
diff --git a/tests/unit/health/test_perf_loop_facts_python.py b/tests/unit/health/test_perf_loop_facts_python.py
index 0a59e5282..058d3f468 100644
--- a/tests/unit/health/test_perf_loop_facts_python.py
+++ b/tests/unit/health/test_perf_loop_facts_python.py
@@ -492,7 +492,8 @@ def test_session_get_sink_shape(call, db_import, sink):
# ---------------------------------------------------------------------------
-def test_batch_session_get_equivalent():
+def test_batch_session_get_is_named_but_never_proven():
+ """``get`` may be an identity-map hit, so batching it is not a proven saving."""
facts = _loop(
b"from sqlalchemy import select\n"
b"async def f(session, rows):\n"
@@ -500,18 +501,7 @@ def test_batch_session_get_equivalent():
b" obj = await session.get(Repo, r.id)\n"
)
assert facts.batch.call == "select(Repo).where(inspect(Repo).primary_key[0].in_(keys))"
- assert facts.batch.equivalent is True
-
-
-def test_batch_session_get_on_the_whole_element_is_not_equivalent():
- """The element itself may be a composite key tuple, which one column's IN does not match."""
- facts = _loop(
- b"from sqlalchemy import select\n"
- b"async def f(session, keys):\n"
- b" for key in keys:\n"
- b" obj = await session.get(Repo, key)\n"
- )
- assert facts.batch is not None and facts.batch.equivalent is False
+ assert facts.batch.equivalent is False
def test_batch_session_get_second_sink_in_body_not_equivalent():
@@ -620,3 +610,131 @@ def test_a_result_read_from_one_chunk_of_keys_does_not_grow():
b" await session.execute(select(H).where(H.id == g.id))\n"
)
assert facts.magnitude != "grows_with_data"
+
+
+# ``.all()`` on a bare name is a query result only when the function binds that name;
+# an imported registry (dispatch's ``plugins.all()``) is not.
+
+
+ ("setup", "receiver_call", "sink"),
+ [
+ ("from dispatch.plugins.base import plugins\n", "plugins.all()", False),
+ ("registry = Registry()\n", "registry.all()", False), # module global, lower-case
+ ("REGISTRY = Registry()\n", "REGISTRY.all()", False), # module global, ALL_CAPS
+ ("", "query.all()", True), # a function parameter
+ ],
+)
+def test_ambiguous_db_method_receiver_gate(setup, receiver_call, sink):
+ src = (
+ "from sqlalchemy import select\n"
+ + setup
+ + "def f(query, ids):\n"
+ + " for i in ids:\n"
+ + f" {receiver_call}\n"
+ )
+ hits = _hits(src.encode())
+ assert [h.detail for h in hits] == (["db"] if sink else [])
+
+
+def test_ambiguous_db_method_on_a_locally_assigned_result_is_still_a_sink():
+ hits = _hits(
+ b"from sqlalchemy import select\n"
+ b"def f(session, ids):\n"
+ b" for i in ids:\n"
+ b" res = session.execute(select(i))\n"
+ b" res.all()\n"
+ )
+ assert [h.detail for h in hits] == ["db", "db"]
+
+
+def test_ambiguous_db_method_on_an_imported_registry_does_not_grow():
+ facts = _loop(
+ b"from sqlalchemy import select\n"
+ b"from dispatch.plugins.base import plugins\n"
+ b"def f(session, ids):\n"
+ b" for p in plugins.all():\n"
+ b" session.execute(select(p))\n"
+ )
+ assert facts is None or facts.magnitude != "grows_with_data"
+
+
+def test_ambiguous_db_method_on_a_locally_assigned_result_still_grows():
+ facts = _loop(
+ b"from sqlalchemy import select\n"
+ b"async def f(session, ids):\n"
+ b" res = await session.execute(select(1))\n"
+ b" for r in res.all():\n"
+ b" session.execute(select(r))\n"
+ )
+ assert facts.magnitude == "grows_with_data"
+
+
[email protected]("stmt", ["run_task(document_id=doc.id)", "t = run_task(doc.id)"])
+def test_a_read_after_a_call_made_for_its_effect_is_not_equivalent(stmt):
+ """dify: the loop runs a task, then reads what it wrote; read up front, it runs first."""
+ facts = _loop(
+ b"from sqlalchemy import select\n"
+ b"def f(session, docs):\n"
+ b" for doc in docs:\n"
+ + f" {stmt}\n".encode()
+ + b" session.scalars(select(Seg).where(Seg.document_id == doc.id)).all()\n"
+ )
+ assert facts.batch is not None and facts.batch.equivalent is False
+
+
+def test_a_local_collection_update_keeps_the_read_equivalent():
+ facts = _loop(
+ b"from sqlalchemy import select\n"
+ b"async def f(session, grants):\n"
+ b" seen = set()\n"
+ b" for g in grants:\n"
+ b" res = await session.execute(select(M).where(M.id == g.member_id))\n"
+ b" key = str(g.member_id)\n"
+ b" seen.add(key)\n"
+ )
+ assert facts.batch is not None and facts.batch.equivalent is True
+
+
+def test_a_raw_sql_read_capped_in_its_text_does_not_grow():
+ """dify: ``sa.text("... LIMIT 1000")`` pages a table; one page is as large as its cap."""
+ facts = _loop(
+ b"import sqlalchemy as sa\n"
+ b"def f(conn, session):\n"
+ b' sql = "SELECT id FROM apps ORDER BY created_at LIMIT 1000"\n'
+ b" rs = conn.execute(sa.text(sql))\n"
+ b" for i in rs:\n"
+ b" session.execute(sa.text('DELETE FROM x WHERE id = :id'), {'id': i.id})\n"
+ )
+ assert facts is None or facts.magnitude != "grows_with_data"
+
+
+def test_a_write_through_a_local_session_before_the_read_is_not_equivalent():
+ facts = _loop(
+ b"from sqlalchemy import select\n"
+ b"def f(rows):\n"
+ b" session = SessionLocal()\n"
+ b" for r in rows:\n"
+ b" session.add(r)\n"
+ b" session.execute(select(M).where(M.id == r.id))\n"
+ )
+ assert facts.batch is not None and facts.batch.equivalent is False
+
+
+ "text",
+ [
+ "note = 'throttle limit 5 per user'", # not SQL
+ "note = 'SELECT id FROM apps -- was LIMIT 1000'", # the cap is commented out
+ ],
+)
+def test_a_limit_outside_live_sql_does_not_cap_the_read(text):
+ facts = _loop(
+ b"import sqlalchemy as sa\n"
+ b"def f(conn, session):\n"
+ + f" {text}\n".encode()
+ + b" rs = conn.execute(sa.text('SELECT id FROM apps'), note)\n"
+ b" for i in rs:\n"
+ b" session.execute(sa.text('DELETE FROM x WHERE id = :id'), {'id': i.id})\n"
+ )
+ assert facts.magnitude == "grows_with_data"