Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(perf): a schema statement or a split of a module string in a loop is not an N+1 (#4363)
Raghav Chamadiya1e922a5b869e
diff --git a/docs/layers/PERFORMANCE.md b/docs/layers/PERFORMANCE.mdindex 1127206ec..106da81c0 100644--- a/docs/layers/PERFORMANCE.md+++ b/docs/layers/PERFORMANCE.md@@ -30,6 +30,10 @@ A few shapes are deliberately quiet:- A TypeScript or JavaScript `includes` check against a short `const` table ofliterals that the file never changes is not flagged, because a table of fixedsize costs the same at any scale.+- A Python schema statement (`CREATE`, `DROP`, `ALTER`, `TRUNCATE`, `PRAGMA` and+ peers) run in a loop is not an N+1: it runs once per table or setting the code+ names, not once per row. A loop over a split of a string the module spells out+ (`for stmt in SCHEMA.split(";")`) is a fixed loop, like one over a literal tuple.- An awaited call on a line that an ESLint `no-await-in-loop` suppression coversis not reported: the comment states that the sequential await is intended.diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.pyindex 47375dd8f..788c5222a 100644--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py@@ -66,11 +66,14 @@ from .python_nodes import (_bindings,_binds,_field_text,+ _first_arg,_fixed_set,_in_filter,_literal_range,_module_int_constant,+ _runs_schema_statement,_scalar_literal,+ _splits_constant,_string_value,_text,_value_refs,@@ -171,7 +174,9 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):io_names=io_names,has_db_import=has_db_import or self.shows_a_query(call),)- if kind == "db" and self._reads_a_registry(call, io_names):+ if kind == "db" and (+ self._reads_a_registry(call, io_names) or _runs_schema_statement(call, self.reaching_value)+ ):return Noneorm_get = self._orm_get(call) if kind is None else Noneif orm_get is not None:@@ -205,8 +210,7 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):def _passes_a_statement(self, call: Node) -> bool:"""The first argument is a statement builder call or a SQL string literal."""- args = call.child_by_field_name("arguments")- first = next((c for c in args.children if c.is_named), None) if args is not None else None+ first = _first_arg(call)if first is None:return Falseif first.type == "call":@@ -278,8 +282,9 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):return model, positional[1]def is_constant_loop(self, node: Node) -> bool:- """``for _ in range(<int literals>)``, ``for x in (<literal>)`` or- ``for x in ALL_CAPS`` (a module constant by convention)."""+ """``for _ in range(<int literals>)``, ``for x in (<literal>)``,+ ``for x in ALL_CAPS`` (a module constant by convention) or a split of a+ string the source spells out (``for stmt in SCHEMA.split(";")``)."""if node.type != "for_statement":return Falseright = node.child_by_field_name("right")@@ -291,7 +296,7 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):name = right.text.decode("utf-8", "replace")if name.isupper() and len(name) > 1:return True- return right.type == "call" and _literal_range(right)+ return right.type == "call" and (_literal_range(right) or _splits_constant(right))# Builtins that wrap a collection without changing what is iterated, so the# same-collection O(n^2) shape sees through them: ``for x in enumerate(items)``diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.pyindex 6e7d4b2c8..81efd5c2b 100644--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py@@ -28,6 +28,13 @@ _PY_SQL_LITERAL_RE = re.compile(r"^[rRuUbBfF]*[\"']+\s*(?:SELECT|INSERT|UPDATE|DELETE|WITH|CREATE|DROP|ALTER|PRAGMA)\b",re.IGNORECASE,)+# Schema statements run once per table, index or setting the code names, however+# many rows those tables hold: a loop of them is no N+1 and has no batch form.+_PY_SCHEMA_SQL_RE = re.compile(+ r"^[rRuUbBfF]*[\"']+\s*(?:CREATE|DROP|ALTER|TRUNCATE|PRAGMA|VACUUM|REINDEX|ANALYZE"+ r"|COMMENT\s+ON|GRANT|REVOKE)\b",+ re.IGNORECASE,+)PY_DB_AMBIGUOUS: frozenset[str] = frozenset({"all", "first", "one", "one_or_none"})PY_SUBPROC_METHODS: frozenset[str] = frozenset({"run", "call", "check_call", "check_output", "Popen"}diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.pyindex 9d51e6da2..7e6ae38af 100644--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py@@ -2,6 +2,7 @@from __future__ import annotations+from collections.abc import Callablefrom typing import TYPE_CHECKINGfrom .base import BasePerfDialect@@ -10,12 +11,39 @@ from .python_lexicon import (_PY_BINDING_WRAPPERS,_PY_CONSTANT_RHS,_PY_DEF_KINDS,+ _PY_SCHEMA_SQL_RE,+ _PY_STRING_KINDS,)if TYPE_CHECKING:from tree_sitter import Node+def _first_arg(call: Node) -> Node | None:+ args = call.child_by_field_name("arguments")+ return next((c for c in args.children if c.is_named), None) if args is not None else None+++def _runs_schema_statement(call: Node, value_of: Callable[[Node, bytes], Node | None]) -> bool:+ """*call*'s statement is DDL or a PRAGMA (``_PY_SCHEMA_SQL_RE``): its first+ argument, the value *value_of* says that name holds, or the text in ``text(...)``.++ Ceiling: Python only, and a module-level statement constant is not followed.+ """+ statement = _first_arg(call)+ if statement is not None and statement.type == "identifier":+ statement = value_of(call, statement.text or b"")+ statement = _text_arg(statement) if statement is not None and statement.type == "call" else statement+ return statement is not None and statement.type in _PY_STRING_KINDS and bool(+ _PY_SCHEMA_SQL_RE.match(_text(statement))+ )+++def _text_arg(call: Node) -> Node | None:+ """The statement in ``text("...")``; any other call is no literal statement."""+ return _first_arg(call) if _field_text(call, "function").rsplit(".", 1)[-1] == "text" else None++def _binds(ident: Node) -> bool:"""Whether *ident* is a name being bound (assigned, imported, a parameter)."""cur, parent = ident, ident.parent@@ -45,8 +73,32 @@ def _bindings(scope: Node, name: bytes) -> list[Node]:return found+_SPLITS = frozenset({"split", "splitlines"})++def _module_int_constant(name: Node) -> bool:- """An ALL_CAPS name its own module binds exactly once, to an integer literal+ return _module_constant(name, lambda value: value.type == "integer")+++def _splits_constant(call: Node) -> bool:+ """``"a b".split()`` or ``SCHEMA.split(";")`` with ``SCHEMA`` a module string+ literal: the source fixes every piece."""+ fn = call.child_by_field_name("function")+ if fn is None or fn.type != "attribute" or _field_text(fn, "attribute") not in _SPLITS:+ return False+ text = fn.child_by_field_name("object")+ return text is not None and (_plain_string(text) or _module_constant(text, _plain_string))+++def _plain_string(node: Node) -> bool:+ """A string literal with no ``{...}`` interpolation, so its text is fixed."""+ return node.type in _PY_STRING_KINDS and not any(+ n.type == "interpolation" for n in BasePerfDialect._walk(node)+ )+++def _module_constant(name: Node, is_literal: Callable[[Node], bool]) -> bool:+ """An ALL_CAPS name its own module binds exactly once, to a value *is_literal* accepts(``MAX = 5``), and no enclosing function rebinds or shadows.The spelling alone proves nothing (``NUM_USERS`` grows), so a name rebound,@@ -63,7 +115,7 @@ def _module_int_constant(name: Node) -> bool:if len(bound) != 1 or bound[0].parent is None or bound[0].parent.type != "assignment":return Falseright = bound[0].parent.child_by_field_name("right")- return right is not None and right.type == "integer"+ return right is not None and is_literal(right)def _unshadowed_module(name: Node, text: bytes) -> Node | None:diff --git a/tests/unit/health/test_perf_io_in_loop.py b/tests/unit/health/test_perf_io_in_loop.pyindex 009e2bd10..99634a45a 100644--- a/tests/unit/health/test_perf_io_in_loop.py+++ b/tests/unit/health/test_perf_io_in_loop.py@@ -115,6 +115,55 @@ _CASES = [[],"ALL_CAPS named-constant collection is skipped",),+ (+ "python",+ b"import sqlite3\n"+ b"def f(conn):\n"+ b" for (t,) in conn.execute('SELECT name FROM sqlite_master').fetchall():\n"+ b" conn.execute(f'DROP TABLE \"{t}\"')\n"+ b" for t in conn.execute('SELECT name FROM t').fetchall():\n"+ b" sql = f'ALTER TABLE {t} ADD COLUMN c'\n"+ b" conn.execute(sql)\n"+ b" conn.execute(text('CREATE INDEX i ON t (c)'))\n",+ [],+ "a schema statement runs once per schema object, not per row",+ ),+ (+ "python",+ b"import sqlite3\n"+ b"def f(conn, ids):\n"+ b" for i in ids:\n"+ b" conn.execute('SELECT * FROM t WHERE id = ?', (i,))\n"+ b" conn.execute(f'UPDATE t SET n = 1 WHERE id = {i}')\n",+ [("io_in_loop", "db"), ("io_in_loop", "db")],+ "a data statement in the same shape stays a sink",+ ),+ (+ "python",+ b"import sqlite3\n"+ b'_SCHEMA = """CREATE TABLE a (x); CREATE TABLE b (y)"""\n'+ b"def f(conn):\n"+ b" for statement in _SCHEMA.split(';'):\n"+ b" conn.execute(statement)\n"+ b" for name in 'a b'.split():\n"+ b" conn.execute(name)\n",+ [],+ "a split of a module string literal is a constant loop",+ ),+ (+ "python",+ b"import sqlite3\n"+ b"_ROWS = load()\n"+ b"def f(conn, text):\n"+ b" for line in _ROWS.split():\n"+ b" conn.execute(line)\n"+ b" for line in text.split():\n"+ b" conn.execute(line)\n"+ b" for line in f'{text}'.split():\n"+ b" conn.execute(line)\n",+ [("io_in_loop", "db"), ("io_in_loop", "db"), ("io_in_loop", "db")],+ "a split of a name not bound to a literal is not constant",+ ),("python",b"from sqlalchemy import select\n"