Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(health): add pandas_iterrows_in_loop performance marker (#542)

Raghav Chamadiya36514c52a4ad
diff --git a/packages/core/src/repowise/core/analysis/health/biomarkers/pandas_iterrows_in_loop.py b/packages/core/src/repowise/core/analysis/health/biomarkers/pandas_iterrows_in_loop.py
new file mode 100644
index 000000000..59676d2f8
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/biomarkers/pandas_iterrows_in_loop.py
@@ -0,0 +1,48 @@
+"""Pandas-iterrows-in-loop — ``for _, row in df.iterrows():``.
+
+``DataFrame.iterrows()`` boxes every row into a fresh ``Series``, making
+row-by-row iteration an order of magnitude slower than a vectorized operation
+(and slower than ``itertuples`` / ``to_dict("records")`` when per-row access is
+unavoidable). The offending call sits in the loop *header*, so the body
+loop-call markers miss it; the walker fires this via the dialect
+``loop_iterable_call_marker`` hook against the loop node. A ``performance``
+dimension signal, gated by the Python dialect to the distinctive ``iterrows``
+method on a member-access receiver. This detector lifts the hits into findings.
+"""
+
+from __future__ import annotations
+
+from ..models import Severity
+from .base import BiomarkerResult, FileContext
+
+_KIND = "pandas_iterrows_in_loop"
+
+
+class PandasIterrowsInLoopDetector:
+ name = _KIND
+ category = "performance"
+
+ def detect(self, ctx: FileContext) -> list[BiomarkerResult]:
+ out: list[BiomarkerResult] = []
+ for hit in ctx.perf_hits:
+ if hit.kind != _KIND:
+ continue
+ out.append(
+ BiomarkerResult(
+ biomarker_type=self.name,
+ severity=Severity.MEDIUM,
+ function_name=hit.function,
+ line_start=hit.line,
+ line_end=hit.line,
+ details={},
+ reason=(
+ "DataFrame.iterrows() boxes each row into a Series (row-by-row, slow); "
+ "prefer a vectorized operation, or itertuples()/to_dict('records') when "
+ "per-row access is unavoidable"
+ ),
+ )
+ )
+ return out
+
+
+BIOMARKER = PandasIterrowsInLoopDetector()
diff --git a/packages/core/src/repowise/core/analysis/health/biomarkers/registry.py b/packages/core/src/repowise/core/analysis/health/biomarkers/registry.py
index a52ebf22a..76bc2ebcd 100644
--- a/packages/core/src/repowise/core/analysis/health/biomarkers/registry.py
+++ b/packages/core/src/repowise/core/analysis/health/biomarkers/registry.py
@@ -47,6 +47,7 @@ from .nested_complexity import NestedComplexityDetector
from .nested_loop_quadratic import NestedLoopQuadraticDetector
from .nested_loop_with_io import NestedLoopWithIoDetector
from .ownership_risk import OwnershipRiskDetector
+from .pandas_iterrows_in_loop import PandasIterrowsInLoopDetector
from .pd_concat_in_loop import PdConcatInLoopDetector
from .primitive_obsession import PrimitiveObsessionDetector
from .prior_defect import PriorDefectDetector
@@ -99,6 +100,7 @@ _DETECTOR_FACTORIES: list[type[Biomarker]] = [
# Phase 7d — language-specific markers (advisory weight; bounded by the cap).
ListInsertZeroInLoopDetector, # type: ignore[list-item]
PdConcatInLoopDetector, # type: ignore[list-item]
+ PandasIterrowsInLoopDetector, # type: ignore[list-item]
JsonParseInLoopDetector, # type: ignore[list-item]
ArraySpreadInReduceDetector, # type: ignore[list-item]
GoroutineInUnboundedLoopDetector, # type: ignore[list-item]
diff --git a/packages/core/src/repowise/core/analysis/health/complexity/walker.py b/packages/core/src/repowise/core/analysis/health/complexity/walker.py
index 638dd4f17..35e9c6a7f 100644
--- a/packages/core/src/repowise/core/analysis/health/complexity/walker.py
+++ b/packages/core/src/repowise/core/analysis/health/complexity/walker.py
@@ -1031,6 +1031,10 @@ _LOOP_STMT_MARKER_KINDS = frozenset(
# Markers for a call that is its OWN iteration construct (``.reduce``), a perf
# smell at any loop depth — emitted via ``dialect.bare_call_marker``.
_BARE_CALL_MARKER_KINDS = frozenset({"array_spread_in_reduce"})
+# Markers fired on a loop whose ITERABLE is itself a slow call (``df.iterrows()``)
+# — the call lives in the loop header, so the body markers miss it. Emitted via
+# ``dialect.loop_iterable_call_marker`` against the loop node.
+_LOOP_ITERABLE_CALL_MARKER_KINDS = frozenset({"pandas_iterrows_in_loop"})
# Boundary kinds that are *unambiguously blocking* when called synchronously, so
# a loop_depth-0 occurrence in a hot function is a ``hot_path_sync_io`` candidate.
@@ -1112,6 +1116,7 @@ def _collect_perf_hits(
do_loop_call_marker = bool(markers & _LOOP_CALL_MARKER_KINDS)
do_loop_stmt_marker = bool(markers & _LOOP_STMT_MARKER_KINDS)
do_bare_call_marker = bool(markers & _BARE_CALL_MARKER_KINDS)
+ do_loop_iterable_call_marker = bool(markers & _LOOP_ITERABLE_CALL_MARKER_KINDS)
do_serial_await = "serial_await_in_loop" in markers
# Phase 7b markers.
do_nested_io = "nested_loop_with_io" in markers
@@ -1193,6 +1198,17 @@ def _collect_perf_hits(
if misc[0] == 0:
misc[0] = node.start_point[0] + 1
+ # A loop whose ITERABLE is itself a slow call (``for _, r in
+ # df.iterrows()``): the call sits in the loop header (runs once), so the
+ # body call markers never see it. Fire on the loop node itself, at any
+ # nesting depth — ``iterrows`` is O(n)-boxing slow on its own.
+ if is_loop and do_loop_iterable_call_marker:
+ icm = dialect.loop_iterable_call_marker(node)
+ if icm is not None:
+ hits.append(
+ PerfHit(icm, node.start_point[0] + 1, next_func, "", func_start=next_start)
+ )
+
if t in call_kinds:
method = dialect.callee_method_name(node) or ""
root_name = dialect.callee_root_name(node) or ""
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py
index 453ee8580..e1cfd6b47 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py
@@ -293,6 +293,19 @@ class BasePerfDialect:
"""
return None
+ def loop_iterable_call_marker(self, node: Node) -> str | None:
+ """Marker kind for a loop whose *iterable expression is itself a call*.
+
+ Used for ``pandas_iterrows_in_loop`` (``for _, row in df.iterrows()``):
+ the offending call sits in the loop HEADER, so it runs once and the body
+ :meth:`loop_call_marker` (loop_depth >= 1) never sees it. This hook is
+ handed the loop node itself, so the dialect inspects the iterable and
+ fires regardless of nesting depth (``iterrows`` is O(n)-boxing slow on
+ its own, not only when nested). Default ``None`` so a language that does
+ not override it is byte-for-byte unchanged.
+ """
+ return None
+
def loop_stmt_marker(self, node: Node, list_names: frozenset[str]) -> str | None:
"""Marker kind for a loop-nested *non-call statement* node.
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
index 8df65f05c..a7ceb0101 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
@@ -111,6 +111,8 @@ class PythonPerfDialect(BasePerfDialect):
# Phase 7d — Python-specific quadratic anti-patterns.
"list_insert_zero_in_loop",
"pd_concat_in_loop",
+ # Header-call marker: the slow iterable IS the loop driver.
+ "pandas_iterrows_in_loop",
}
)
@@ -325,6 +327,27 @@ class PythonPerfDialect(BasePerfDialect):
return "pd_concat_in_loop"
return None
+ def loop_iterable_call_marker(self, node: Node) -> str | None:
+ """``for _, row in df.iterrows():`` — row-by-row DataFrame iteration.
+
+ ``DataFrame.iterrows()`` boxes every row into a fresh Series, an order of
+ magnitude slower than a vectorized operation (and slower than
+ ``itertuples`` / ``to_dict`` when row access is unavoidable). The call
+ sits in the loop header, so the body ``loop_call_marker`` misses it.
+ Gated to the distinctive method name on a member-access receiver
+ (``x.iterrows()``, never a bare ``iterrows(...)``) — high-precision by
+ construction, like the ``pd``/``pandas`` ``concat`` root: ``iterrows`` is
+ a pandas-only verb with no common collision.
+ """
+ if node.type != "for_statement":
+ return None
+ right = node.child_by_field_name("right")
+ if right is None or right.type != "call":
+ return None
+ if self.callee_method_name(right) == "iterrows" and self.callee_is_attribute(right):
+ return "pandas_iterrows_in_loop"
+ return None
+
def _receiver_reset_in_loop(self, node: Node, name: str) -> bool:
"""True if the call's receiver ``name`` is re-assigned (reset to a fresh
list) inside an enclosing loop body — so the per-front-insert is bounded
diff --git a/packages/core/src/repowise/core/analysis/health/scoring.py b/packages/core/src/repowise/core/analysis/health/scoring.py
index 6fcccdaf6..104883a00 100644
--- a/packages/core/src/repowise/core/analysis/health/scoring.py
+++ b/packages/core/src/repowise/core/analysis/health/scoring.py
@@ -225,6 +225,7 @@ _BIOMARKER_DIMENSIONS: dict[str, set[str]] = {
# Phase 7d language-specific markers - performance-only.
"list_insert_zero_in_loop": {"performance"},
"pd_concat_in_loop": {"performance"},
+ "pandas_iterrows_in_loop": {"performance"},
"json_parse_in_loop": {"performance"},
"array_spread_in_reduce": {"performance"},
"goroutine_in_unbounded_loop": {"performance"},
@@ -324,6 +325,10 @@ _PERFORMANCE_WEIGHT_MULTIPLIER: dict[str, float] = {
# precision json_parse / goroutine-spawn ones.
"list_insert_zero_in_loop": 0.6,
"pd_concat_in_loop": 0.6,
+ # ``iterrows`` is a documented pandas anti-pattern, high-precision by the
+ # distinctive method name; advisory pending corpus volume (no pandas in the
+ # OSS gate corpus — by-construction, like pd_concat).
+ "pandas_iterrows_in_loop": 0.6,
"array_spread_in_reduce": 0.5,
"json_parse_in_loop": 0.4,
"goroutine_in_unbounded_loop": 0.4,
@@ -345,6 +350,7 @@ _PERFORMANCE_CATEGORY: dict[str, str] = {
"blocking_io_under_lock": "performance",
"list_insert_zero_in_loop": "performance",
"pd_concat_in_loop": "performance",
+ "pandas_iterrows_in_loop": "performance",
"json_parse_in_loop": "performance",
"array_spread_in_reduce": "performance",
"goroutine_in_unbounded_loop": "performance",
@@ -371,6 +377,7 @@ _PERFORMANCE_HOME: frozenset[str] = frozenset(
"blocking_io_under_lock",
"list_insert_zero_in_loop",
"pd_concat_in_loop",
+ "pandas_iterrows_in_loop",
"json_parse_in_loop",
"array_spread_in_reduce",
"goroutine_in_unbounded_loop",
diff --git a/tests/unit/health/test_perf_dialects.py b/tests/unit/health/test_perf_dialects.py
index e64a98f9e..bba556f19 100644
--- a/tests/unit/health/test_perf_dialects.py
+++ b/tests/unit/health/test_perf_dialects.py
@@ -97,7 +97,7 @@ def test_java_cases(src, expected, note):
def test_go_fixture_counts():
fc = _walk("go/perf_io_in_loop.go", "go")
counts = _kinds(fc.perf_hits)
- # db (db.Query) · network (http.Get) · filesystem (os.Open ×2, incl. the
+ # db (db.Query) · network (http.Get) · filesystem (os.Open x2, incl. the
# one in deferInLoop). The constant-bound and out-of-loop queries do not.
assert counts["io_in_loop"] == 4
assert {h.detail for h in fc.perf_hits if h.kind == "io_in_loop"} == {
@@ -382,6 +382,19 @@ def test_python_pd_concat_in_loop():
assert ("pd_concat_in_loop", "") in _hits("python", src)
+def test_python_pandas_iterrows_in_loop():
+ # The iterrows() call lives in the loop HEADER, so the body call-markers
+ # never see it — the loop_iterable_call_marker hook fires on the loop node.
+ iterrows = "def f(df):\n for _, row in df.iterrows():\n use(row)\n"
+ assert ("pandas_iterrows_in_loop", "") in _hits("python", iterrows)
+ # itertuples is the recommended faster alternative — never flagged.
+ tuples = "def f(df):\n for row in df.itertuples():\n use(row)\n"
+ assert not any(k == "pandas_iterrows_in_loop" for k, _ in _hits("python", tuples))
+ # A plain collection iterable is not a header-call smell.
+ plain = "def f(rows):\n for row in rows:\n use(row)\n"
+ assert not any(k == "pandas_iterrows_in_loop" for k, _ in _hits("python", plain))
+
+
def test_ts_json_parse_in_loop():
src = "function f(xs){ for (const x of xs) { const c = JSON.parse(JSON.stringify(x)); } }"
assert ("json_parse_in_loop", "") in _hits("typescript", src)