Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(perf): prove fewer batch plans, and stop reading a plugin registry as a query (#2521)
Raghav Chamadiya69b167e25bfd
diff --git a/packages/core/src/repowise/core/analysis/health/engine.py b/packages/core/src/repowise/core/analysis/health/engine.pyindex 093dfa3c5..daa51d080 100644--- a/packages/core/src/repowise/core/analysis/health/engine.py+++ b/packages/core/src/repowise/core/analysis/health/engine.py@@ -111,6 +111,10 @@ log = structlog.get_logger(__name__)# and a declining call's arguments are not scanned, so an assertion passed as an# argument still does not stand in for the header's oracle.#+# v26: ``.all()`` on an imported or module-global name (a plugin registry) is not a+# db sink, a SQL string saying ``LIMIT n`` caps a read, and a batched read is never+# proven for ``session.get`` or beside a call made for its effect; a v25 store differs.+## v25: repetition with nothing to change (a filesystem or subprocess boundary, or# a loop already walking chunks) is stored as ``expected``; a v24 store says investigate.#@@ -253,7 +257,7 @@ log = structlog.get_logger(__name__)# forms. Files that were counted untested and are not become tested, which# moves untested-hotspot findings and the scores that carry them, on every# language with a prefix or spec convention rather than Ruby alone.-HEALTH_ANALYZER_VERSION = 25+HEALTH_ANALYZER_VERSION = 26def walked_functions(diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.pyindex df2964b47..41c7f11ad 100644--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py@@ -8,6 +8,7 @@ zero Python behavior — the defect golden + perf suite lock that.from __future__ import annotations+import builtinsimport refrom typing import TYPE_CHECKING, ClassVar@@ -171,6 +172,9 @@ _PY_RESULT_PROJECTIONS: frozenset[str] = frozenset({"all", "scalars", "fetchall", "json", "data", "rows"})_PY_BOUND_NAME_RE = re.compile(r"(?i)retr|attempt")+# Raw SQL capped in its own text: ``sa.text("... LIMIT 1000")``.+_PY_SQL_LIMIT_RE = re.compile(rb"(?is)\bselect\b.*\blimit\s+\d+\b")+_PY_SQL_COMMENT_RE = re.compile(rb"--[^\n]*|/\*.*?\*/", re.S)# A per-key call that limits or orders its rows is not the read one IN query makes._PY_PER_KEY_LIMITS: frozenset[str] = frozenset({@@ -182,6 +186,7 @@ _PY_ONE_ROW: frozenset[str] = _PY_PER_KEY_LIMITS | {"scalar"}_PY_READ_CAPS: frozenset[str] = _PY_ONE_ROW - {"order", "order_by", "offset"}_PY_WRITES: frozenset[str] = frozenset({"update", "insert", "upsert", "values"})_PY_SCOPES: frozenset[str] = frozenset({"function_definition", "lambda", "class_definition"})+_PY_BUILTINS: frozenset[str] = frozenset(dir(builtins))_PY_SEM_NAME_RE = re.compile(r"(?i)(sem|semaphore|limiter|limit)$")# A model class is CapWords; an ALL_CAPS constant (``API_URL``) is not._PY_MODEL_NAME_RE = re.compile(r"^[A-Z]\w*[a-z]\w*$")@@ -293,6 +298,8 @@ class PythonPerfDialect(BasePerfDialect):kind = super().call_sink_kind(call, awaited=awaited, io_names=io_names, has_db_import=has_db_import)+ if kind == "db" and self._reads_a_registry(call, io_names):+ return Noneorm_get = self._orm_get(call) if kind is None else Noneif orm_get is not None:# ``get`` alone is ``dict.get`` / ``requests.get``; the model-class shape@@ -301,6 +308,28 @@ class PythonPerfDialect(BasePerfDialect):return "db" if db and not self._key_used_earlier(call, orm_get[1]) else Nonereturn kind+ def _reads_a_registry(self, call: Node, io_names: dict[str, str]) -> bool:+ """``plugins.all()`` on an imported or module-global name reads a registry, not a+ query result: a result is bound in the function that reads it."""+ if self.callee_method_name(call) not in PY_DB_AMBIGUOUS:+ return False+ fn = call.child_by_field_name("function")+ receiver = fn.child_by_field_name("object") if fn is not None else None+ if receiver is None or receiver.type != "identifier":+ return False+ name = receiver.text or b""+ return io_names.get(name.decode()) != "db" and not self._bound_locally(call, name)++ def _bound_locally(self, call: Node, name: bytes) -> bool:+ """*name* is assigned before *call* in its function, or is a parameter of it."""+ if self._reaching_rhs(call, name) is not None:+ return True+ scope = call.parent+ while scope is not None and scope.type != "function_definition":+ scope = scope.parent+ params = scope.child_by_field_name("parameters") if scope is not None else None+ return params is not None and name.decode() in self._param_names(params)+def _key_used_earlier(self, call: Node, key: Node) -> bool:"""An earlier call in this loop body gave the same session the same key, so therow may already sit in the session and ``get`` answers from memory."""@@ -814,12 +843,18 @@ class PythonPerfDialect(BasePerfDialect):return Nonedef _caps_read(self, node: Node, loop: Node) -> bool:- """``.limit(n)`` and peers, or ``.in_()`` over one chunk of keys- (``xs[i:i + N]``, or the element of a loop that walks ``xs`` in chunks)."""+ """``.limit(n)`` and peers, a SQL string saying ``LIMIT n``, or ``.in_()`` over+ one chunk of keys."""+ if node.type == "string":+ return bool(_PY_SQL_LIMIT_RE.search(_PY_SQL_COMMENT_RE.sub(b"", node.text or b"")))method = self.callee_method_name(node) if node.type == "call" else Noneif method in _PY_READ_CAPS:return True- args = node.child_by_field_name("arguments") if method == "in_" else None+ return method == "in_" and self._in_one_chunk(node, loop)++ def _in_one_chunk(self, node: Node, loop: Node) -> bool:+ """``.in_(xs[i:i + N])``, or ``.in_(chunk)`` where a loop walks ``xs`` in chunks."""+ args = node.child_by_field_name("arguments")keys = next((c for c in args.children if c.is_named), None) if args else Noneif keys is None:return False@@ -920,19 +955,49 @@ class PythonPerfDialect(BasePerfDialect):kept = self._unconditional(sink, body)else:kept = not self._builds_in_order(body)- orm_get = self._orm_get(sink)- if orm_get is not None and orm_get[1] == refs[0]:- # The whole element as the key may be a composite (tuple) key, which- # ``primary_key[0].in_`` does not match; only ``r.id`` names one column.+ if self._orm_get(sink) is not None:+ # ``get`` answers from the session's identity map when the row is already+ # loaded, which no syntax shows: held out, half of these sites made no+ # round trip. The bulk form is named, never proven.kept = Falseequivalent = (keptand not methods & _PY_PER_KEY_LIMITSand not self._limited_downstream(sink, body)and self._only_io_in_body(sink, body, probe)+ and not self._calls_for_effect(sink, body))return BatchForm(call, equivalent)+ def _calls_for_effect(self, sink: Node, body: Node) -> bool:+ """Another statement calls something this function did not build as a list, set+ or dict (``run_task(doc)``, ``session.add(r)``, ``t = task.delay(d)``), which may+ write what the batched read would read before it ran."""+ for node in self._walk(body, prune=_PY_SCOPES):+ if node.type == "expression_statement":+ call, assigned = node.named_children[0], False+ elif node.type == "assignment":+ call, assigned = node.child_by_field_name("right"), True+ else:+ continue+ if call is not None and call.type == "await":+ call = next((c for c in call.children if c.is_named), None)+ if call is None or call.type != "call" or self._within(call, sink):+ continue+ if self._has_effect(call, assigned=assigned):+ return True+ return False++ def _has_effect(self, call: Node, *, assigned: bool) -> bool:+ """Not ``seen.add(x)`` on a local collection, and, when its result is kept, not a+ builtin (``str(r.id)``) or a method on a local (``res.first()``)."""+ if not self.callee_is_attribute(call):+ return not (assigned and self.callee_method_name(call) in _PY_BUILTINS)+ rhs = self._reaching_rhs(call, (self.callee_root_name(call) or "").encode())+ if rhs is None:+ return True+ return not (assigned or self._rhs_is_list(rhs) or self._rhs_is_nonlist_container(rhs))+def _limited_downstream(self, sink: Node, body: Node) -> bool:"""The result is cut to one row after the call (``(await q).first()``,``res = await q`` then ``res.scalar_one()``), which one IN query does not do."""diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.pyindex 23eb95efd..4e83cf66a 100644--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/ts_js.py@@ -476,12 +476,11 @@ class TsJsPerfDialect(BasePerfDialect):return self._magnitude_of_expr(self._reaching_assignment(loop, right.text), probe)return magnitude- def _limiter_of(self, closure: Node) -> str | None:- """The limiter an awaited call runs *closure* through (``await limit(() => f(x))``).+ def _awaited_wrapper_call(self, closure: Node) -> Node | None:+ """The awaited call *closure* is passed to (``await limit(() => f(x))``).Awaited, the closure finishes before the loop moves on, so it is the loop- body. Stored, returned or not awaited, it runs later and is not. A curried- ``pLimit(2)(...)`` builds a fresh limiter per call, which bounds nothing.+ body. Stored, returned or not awaited, it runs later and is not."""args = closure.parentif closure.type != "arrow_function" or args is None or args.type != "arguments":@@ -492,9 +491,18 @@ class TsJsPerfDialect(BasePerfDialect):fn = call.child_by_field_name("function")if fn is None or fn.type not in ("identifier", "member_expression"):return None+ return call++ def _limiter_of(self, closure: Node) -> str | None:+ """The limiter *closure* runs through. A curried ``pLimit(2)(...)`` builds a+ fresh limiter per call, which bounds nothing, so the callee must be named one."""+ call = self._awaited_wrapper_call(closure)+ if call is None:+ return Nonename = self.callee_method_name(call) or ""if not (_LIMITER_NAME_RE.search(name) or name in _LIMITER_METHODS):return None+ fn = call.child_by_field_name("function")return (fn.text or b"").decode() or Nonedef runs_in_place(self, closure: Node) -> bool:diff --git a/tests/unit/health/test_model_version_coupling.py b/tests/unit/health/test_model_version_coupling.pyindex 8a0ce256b..da6a3d0c2 100644--- a/tests/unit/health/test_model_version_coupling.py+++ b/tests/unit/health/test_model_version_coupling.py@@ -22,7 +22,7 @@ from repowise.core.analysis.health.refactoring.identity import (# The three stamps as they stand together. Moving either identity model means# moving the analyzer with it, and updating this tuple in the same commit.-_STAMPS = (25, 2, 2)+_STAMPS = (26, 2, 2)def test_the_identity_models_and_the_analyzer_stamp_are_pinned_together() -> None:diff --git a/tests/unit/health/test_perf_loop_facts_python.py b/tests/unit/health/test_perf_loop_facts_python.pyindex 0a59e5282..058d3f468 100644--- a/tests/unit/health/test_perf_loop_facts_python.py+++ b/tests/unit/health/test_perf_loop_facts_python.py@@ -492,7 +492,8 @@ def test_session_get_sink_shape(call, db_import, sink):# ----------------------------------------------------------------------------def test_batch_session_get_equivalent():+def test_batch_session_get_is_named_but_never_proven():+ """``get`` may be an identity-map hit, so batching it is not a proven saving."""facts = _loop(b"from sqlalchemy import select\n"b"async def f(session, rows):\n"@@ -500,18 +501,7 @@ def test_batch_session_get_equivalent():b" obj = await session.get(Repo, r.id)\n")assert facts.batch.call == "select(Repo).where(inspect(Repo).primary_key[0].in_(keys))"- assert facts.batch.equivalent is True---def test_batch_session_get_on_the_whole_element_is_not_equivalent():- """The element itself may be a composite key tuple, which one column's IN does not match."""- facts = _loop(- b"from sqlalchemy import select\n"- b"async def f(session, keys):\n"- b" for key in keys:\n"- b" obj = await session.get(Repo, key)\n"- )- assert facts.batch is not None and facts.batch.equivalent is False+ assert facts.batch.equivalent is Falsedef test_batch_session_get_second_sink_in_body_not_equivalent():@@ -620,3 +610,131 @@ def test_a_result_read_from_one_chunk_of_keys_does_not_grow():b" await session.execute(select(H).where(H.id == g.id))\n")assert facts.magnitude != "grows_with_data"+++# ``.all()`` on a bare name is a query result only when the function binds that name;+# an imported registry (dispatch's ``plugins.all()``) is not.+++ ("setup", "receiver_call", "sink"),+ [+ ("from dispatch.plugins.base import plugins\n", "plugins.all()", False),+ ("registry = Registry()\n", "registry.all()", False), # module global, lower-case+ ("REGISTRY = Registry()\n", "REGISTRY.all()", False), # module global, ALL_CAPS+ ("", "query.all()", True), # a function parameter+ ],+)+def test_ambiguous_db_method_receiver_gate(setup, receiver_call, sink):+ src = (+ "from sqlalchemy import select\n"+ + setup+ + "def f(query, ids):\n"+ + " for i in ids:\n"+ + f" {receiver_call}\n"+ )+ hits = _hits(src.encode())+ assert [h.detail for h in hits] == (["db"] if sink else [])+++def test_ambiguous_db_method_on_a_locally_assigned_result_is_still_a_sink():+ hits = _hits(+ b"from sqlalchemy import select\n"+ b"def f(session, ids):\n"+ b" for i in ids:\n"+ b" res = session.execute(select(i))\n"+ b" res.all()\n"+ )+ assert [h.detail for h in hits] == ["db", "db"]+++def test_ambiguous_db_method_on_an_imported_registry_does_not_grow():+ facts = _loop(+ b"from sqlalchemy import select\n"+ b"from dispatch.plugins.base import plugins\n"+ b"def f(session, ids):\n"+ b" for p in plugins.all():\n"+ b" session.execute(select(p))\n"+ )+ assert facts is None or facts.magnitude != "grows_with_data"+++def test_ambiguous_db_method_on_a_locally_assigned_result_still_grows():+ facts = _loop(+ b"from sqlalchemy import select\n"+ b"async def f(session, ids):\n"+ b" res = await session.execute(select(1))\n"+ b" for r in res.all():\n"+ b" session.execute(select(r))\n"+ )+ assert facts.magnitude == "grows_with_data"++[email protected]("stmt", ["run_task(document_id=doc.id)", "t = run_task(doc.id)"])+def test_a_read_after_a_call_made_for_its_effect_is_not_equivalent(stmt):+ """dify: the loop runs a task, then reads what it wrote; read up front, it runs first."""+ facts = _loop(+ b"from sqlalchemy import select\n"+ b"def f(session, docs):\n"+ b" for doc in docs:\n"+ + f" {stmt}\n".encode()+ + b" session.scalars(select(Seg).where(Seg.document_id == doc.id)).all()\n"+ )+ assert facts.batch is not None and facts.batch.equivalent is False+++def test_a_local_collection_update_keeps_the_read_equivalent():+ facts = _loop(+ b"from sqlalchemy import select\n"+ b"async def f(session, grants):\n"+ b" seen = set()\n"+ b" for g in grants:\n"+ b" res = await session.execute(select(M).where(M.id == g.member_id))\n"+ b" key = str(g.member_id)\n"+ b" seen.add(key)\n"+ )+ assert facts.batch is not None and facts.batch.equivalent is True+++def test_a_raw_sql_read_capped_in_its_text_does_not_grow():+ """dify: ``sa.text("... LIMIT 1000")`` pages a table; one page is as large as its cap."""+ facts = _loop(+ b"import sqlalchemy as sa\n"+ b"def f(conn, session):\n"+ b' sql = "SELECT id FROM apps ORDER BY created_at LIMIT 1000"\n'+ b" rs = conn.execute(sa.text(sql))\n"+ b" for i in rs:\n"+ b" session.execute(sa.text('DELETE FROM x WHERE id = :id'), {'id': i.id})\n"+ )+ assert facts is None or facts.magnitude != "grows_with_data"+++def test_a_write_through_a_local_session_before_the_read_is_not_equivalent():+ facts = _loop(+ b"from sqlalchemy import select\n"+ b"def f(rows):\n"+ b" session = SessionLocal()\n"+ b" for r in rows:\n"+ b" session.add(r)\n"+ b" session.execute(select(M).where(M.id == r.id))\n"+ )+ assert facts.batch is not None and facts.batch.equivalent is False+++ "text",+ [+ "note = 'throttle limit 5 per user'", # not SQL+ "note = 'SELECT id FROM apps -- was LIMIT 1000'", # the cap is commented out+ ],+)+def test_a_limit_outside_live_sql_does_not_cap_the_read(text):+ facts = _loop(+ b"import sqlalchemy as sa\n"+ b"def f(conn, session):\n"+ + f" {text}\n".encode()+ + b" rs = conn.execute(sa.text('SELECT id FROM apps'), note)\n"+ b" for i in rs:\n"+ b" session.execute(sa.text('DELETE FROM x WHERE id = :id'), {'id': i.id})\n"+ )+ assert facts.magnitude == "grows_with_data"