Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(perf): a schema statement or a split of a module string in a loop is not an N+1 (#4363)

Raghav Chamadiya1e922a5b869e
diff --git a/docs/layers/PERFORMANCE.md b/docs/layers/PERFORMANCE.md
index 1127206ec..106da81c0 100644
--- a/docs/layers/PERFORMANCE.md
+++ b/docs/layers/PERFORMANCE.md
@@ -30,6 +30,10 @@ A few shapes are deliberately quiet:
- A TypeScript or JavaScript `includes` check against a short `const` table of
literals that the file never changes is not flagged, because a table of fixed
size costs the same at any scale.
+- A Python schema statement (`CREATE`, `DROP`, `ALTER`, `TRUNCATE`, `PRAGMA` and
+ peers) run in a loop is not an N+1: it runs once per table or setting the code
+ names, not once per row. A loop over a split of a string the module spells out
+ (`for stmt in SCHEMA.split(";")`) is a fixed loop, like one over a literal tuple.
- An awaited call on a line that an ESLint `no-await-in-loop` suppression covers
is not reported: the comment states that the sequential await is intended.
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
index 47375dd8f..788c5222a 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
@@ -66,11 +66,14 @@ from .python_nodes import (
_bindings,
_binds,
_field_text,
+ _first_arg,
_fixed_set,
_in_filter,
_literal_range,
_module_int_constant,
+ _runs_schema_statement,
_scalar_literal,
+ _splits_constant,
_string_value,
_text,
_value_refs,
@@ -171,7 +174,9 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):
io_names=io_names,
has_db_import=has_db_import or self.shows_a_query(call),
)
- if kind == "db" and self._reads_a_registry(call, io_names):
+ if kind == "db" and (
+ self._reads_a_registry(call, io_names) or _runs_schema_statement(call, self.reaching_value)
+ ):
return None
orm_get = self._orm_get(call) if kind is None else None
if orm_get is not None:
@@ -205,8 +210,7 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):
def _passes_a_statement(self, call: Node) -> bool:
"""The first argument is a statement builder call or a SQL string literal."""
- args = call.child_by_field_name("arguments")
- first = next((c for c in args.children if c.is_named), None) if args is not None else None
+ first = _first_arg(call)
if first is None:
return False
if first.type == "call":
@@ -278,8 +282,9 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):
return model, positional[1]
def is_constant_loop(self, node: Node) -> bool:
- """``for _ in range(<int literals>)``, ``for x in (<literal>)`` or
- ``for x in ALL_CAPS`` (a module constant by convention)."""
+ """``for _ in range(<int literals>)``, ``for x in (<literal>)``,
+ ``for x in ALL_CAPS`` (a module constant by convention) or a split of a
+ string the source spells out (``for stmt in SCHEMA.split(";")``)."""
if node.type != "for_statement":
return False
right = node.child_by_field_name("right")
@@ -291,7 +296,7 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):
name = right.text.decode("utf-8", "replace")
if name.isupper() and len(name) > 1:
return True
- return right.type == "call" and _literal_range(right)
+ return right.type == "call" and (_literal_range(right) or _splits_constant(right))
# Builtins that wrap a collection without changing what is iterated, so the
# same-collection O(n^2) shape sees through them: ``for x in enumerate(items)``
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py
index 6e7d4b2c8..81efd5c2b 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py
@@ -28,6 +28,13 @@ _PY_SQL_LITERAL_RE = re.compile(
r"^[rRuUbBfF]*[\"']+\s*(?:SELECT|INSERT|UPDATE|DELETE|WITH|CREATE|DROP|ALTER|PRAGMA)\b",
re.IGNORECASE,
)
+# Schema statements run once per table, index or setting the code names, however
+# many rows those tables hold: a loop of them is no N+1 and has no batch form.
+_PY_SCHEMA_SQL_RE = re.compile(
+ r"^[rRuUbBfF]*[\"']+\s*(?:CREATE|DROP|ALTER|TRUNCATE|PRAGMA|VACUUM|REINDEX|ANALYZE"
+ r"|COMMENT\s+ON|GRANT|REVOKE)\b",
+ re.IGNORECASE,
+)
PY_DB_AMBIGUOUS: frozenset[str] = frozenset({"all", "first", "one", "one_or_none"})
PY_SUBPROC_METHODS: frozenset[str] = frozenset(
{"run", "call", "check_call", "check_output", "Popen"}
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py
index 9d51e6da2..7e6ae38af 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_nodes.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from collections.abc import Callable
from typing import TYPE_CHECKING
from .base import BasePerfDialect
@@ -10,12 +11,39 @@ from .python_lexicon import (
_PY_BINDING_WRAPPERS,
_PY_CONSTANT_RHS,
_PY_DEF_KINDS,
+ _PY_SCHEMA_SQL_RE,
+ _PY_STRING_KINDS,
)
if TYPE_CHECKING:
from tree_sitter import Node
+def _first_arg(call: Node) -> Node | None:
+ args = call.child_by_field_name("arguments")
+ return next((c for c in args.children if c.is_named), None) if args is not None else None
+
+
+def _runs_schema_statement(call: Node, value_of: Callable[[Node, bytes], Node | None]) -> bool:
+ """*call*'s statement is DDL or a PRAGMA (``_PY_SCHEMA_SQL_RE``): its first
+ argument, the value *value_of* says that name holds, or the text in ``text(...)``.
+
+ Ceiling: Python only, and a module-level statement constant is not followed.
+ """
+ statement = _first_arg(call)
+ if statement is not None and statement.type == "identifier":
+ statement = value_of(call, statement.text or b"")
+ statement = _text_arg(statement) if statement is not None and statement.type == "call" else statement
+ return statement is not None and statement.type in _PY_STRING_KINDS and bool(
+ _PY_SCHEMA_SQL_RE.match(_text(statement))
+ )
+
+
+def _text_arg(call: Node) -> Node | None:
+ """The statement in ``text("...")``; any other call is no literal statement."""
+ return _first_arg(call) if _field_text(call, "function").rsplit(".", 1)[-1] == "text" else None
+
+
def _binds(ident: Node) -> bool:
"""Whether *ident* is a name being bound (assigned, imported, a parameter)."""
cur, parent = ident, ident.parent
@@ -45,8 +73,32 @@ def _bindings(scope: Node, name: bytes) -> list[Node]:
return found
+_SPLITS = frozenset({"split", "splitlines"})
+
+
def _module_int_constant(name: Node) -> bool:
- """An ALL_CAPS name its own module binds exactly once, to an integer literal
+ return _module_constant(name, lambda value: value.type == "integer")
+
+
+def _splits_constant(call: Node) -> bool:
+ """``"a b".split()`` or ``SCHEMA.split(";")`` with ``SCHEMA`` a module string
+ literal: the source fixes every piece."""
+ fn = call.child_by_field_name("function")
+ if fn is None or fn.type != "attribute" or _field_text(fn, "attribute") not in _SPLITS:
+ return False
+ text = fn.child_by_field_name("object")
+ return text is not None and (_plain_string(text) or _module_constant(text, _plain_string))
+
+
+def _plain_string(node: Node) -> bool:
+ """A string literal with no ``{...}`` interpolation, so its text is fixed."""
+ return node.type in _PY_STRING_KINDS and not any(
+ n.type == "interpolation" for n in BasePerfDialect._walk(node)
+ )
+
+
+def _module_constant(name: Node, is_literal: Callable[[Node], bool]) -> bool:
+ """An ALL_CAPS name its own module binds exactly once, to a value *is_literal* accepts
(``MAX = 5``), and no enclosing function rebinds or shadows.
The spelling alone proves nothing (``NUM_USERS`` grows), so a name rebound,
@@ -63,7 +115,7 @@ def _module_int_constant(name: Node) -> bool:
if len(bound) != 1 or bound[0].parent is None or bound[0].parent.type != "assignment":
return False
right = bound[0].parent.child_by_field_name("right")
- return right is not None and right.type == "integer"
+ return right is not None and is_literal(right)
def _unshadowed_module(name: Node, text: bytes) -> Node | None:
diff --git a/tests/unit/health/test_perf_io_in_loop.py b/tests/unit/health/test_perf_io_in_loop.py
index 009e2bd10..99634a45a 100644
--- a/tests/unit/health/test_perf_io_in_loop.py
+++ b/tests/unit/health/test_perf_io_in_loop.py
@@ -115,6 +115,55 @@ _CASES = [
[],
"ALL_CAPS named-constant collection is skipped",
),
+ (
+ "python",
+ b"import sqlite3\n"
+ b"def f(conn):\n"
+ b" for (t,) in conn.execute('SELECT name FROM sqlite_master').fetchall():\n"
+ b" conn.execute(f'DROP TABLE \"{t}\"')\n"
+ b" for t in conn.execute('SELECT name FROM t').fetchall():\n"
+ b" sql = f'ALTER TABLE {t} ADD COLUMN c'\n"
+ b" conn.execute(sql)\n"
+ b" conn.execute(text('CREATE INDEX i ON t (c)'))\n",
+ [],
+ "a schema statement runs once per schema object, not per row",
+ ),
+ (
+ "python",
+ b"import sqlite3\n"
+ b"def f(conn, ids):\n"
+ b" for i in ids:\n"
+ b" conn.execute('SELECT * FROM t WHERE id = ?', (i,))\n"
+ b" conn.execute(f'UPDATE t SET n = 1 WHERE id = {i}')\n",
+ [("io_in_loop", "db"), ("io_in_loop", "db")],
+ "a data statement in the same shape stays a sink",
+ ),
+ (
+ "python",
+ b"import sqlite3\n"
+ b'_SCHEMA = """CREATE TABLE a (x); CREATE TABLE b (y)"""\n'
+ b"def f(conn):\n"
+ b" for statement in _SCHEMA.split(';'):\n"
+ b" conn.execute(statement)\n"
+ b" for name in 'a b'.split():\n"
+ b" conn.execute(name)\n",
+ [],
+ "a split of a module string literal is a constant loop",
+ ),
+ (
+ "python",
+ b"import sqlite3\n"
+ b"_ROWS = load()\n"
+ b"def f(conn, text):\n"
+ b" for line in _ROWS.split():\n"
+ b" conn.execute(line)\n"
+ b" for line in text.split():\n"
+ b" conn.execute(line)\n"
+ b" for line in f'{text}'.split():\n"
+ b" conn.execute(line)\n",
+ [("io_in_loop", "db"), ("io_in_loop", "db"), ("io_in_loop", "db")],
+ "a split of a name not bound to a literal is not constant",
+ ),
(
"python",
b"from sqlalchemy import select\n"