Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(health): add pandas_iterrows_in_loop performance marker (#542)
Raghav Chamadiya36514c52a4ad
diff --git a/packages/core/src/repowise/core/analysis/health/biomarkers/pandas_iterrows_in_loop.py b/packages/core/src/repowise/core/analysis/health/biomarkers/pandas_iterrows_in_loop.pynew file mode 100644index 000000000..59676d2f8--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/biomarkers/pandas_iterrows_in_loop.py@@ -0,0 +1,48 @@+"""Pandas-iterrows-in-loop — ``for _, row in df.iterrows():``.++``DataFrame.iterrows()`` boxes every row into a fresh ``Series``, making+row-by-row iteration an order of magnitude slower than a vectorized operation+(and slower than ``itertuples`` / ``to_dict("records")`` when per-row access is+unavoidable). The offending call sits in the loop *header*, so the body+loop-call markers miss it; the walker fires this via the dialect+``loop_iterable_call_marker`` hook against the loop node. A ``performance``+dimension signal, gated by the Python dialect to the distinctive ``iterrows``+method on a member-access receiver. This detector lifts the hits into findings.+"""++from __future__ import annotations++from ..models import Severity+from .base import BiomarkerResult, FileContext++_KIND = "pandas_iterrows_in_loop"+++class PandasIterrowsInLoopDetector:+ name = _KIND+ category = "performance"++ def detect(self, ctx: FileContext) -> list[BiomarkerResult]:+ out: list[BiomarkerResult] = []+ for hit in ctx.perf_hits:+ if hit.kind != _KIND:+ continue+ out.append(+ BiomarkerResult(+ biomarker_type=self.name,+ severity=Severity.MEDIUM,+ function_name=hit.function,+ line_start=hit.line,+ line_end=hit.line,+ details={},+ reason=(+ "DataFrame.iterrows() boxes each row into a Series (row-by-row, slow); "+ "prefer a vectorized operation, or itertuples()/to_dict('records') when "+ "per-row access is unavoidable"+ ),+ )+ )+ return out+++BIOMARKER = PandasIterrowsInLoopDetector()diff --git a/packages/core/src/repowise/core/analysis/health/biomarkers/registry.py b/packages/core/src/repowise/core/analysis/health/biomarkers/registry.pyindex a52ebf22a..76bc2ebcd 100644--- a/packages/core/src/repowise/core/analysis/health/biomarkers/registry.py+++ b/packages/core/src/repowise/core/analysis/health/biomarkers/registry.py@@ -47,6 +47,7 @@ from .nested_complexity import NestedComplexityDetectorfrom .nested_loop_quadratic import NestedLoopQuadraticDetectorfrom .nested_loop_with_io import NestedLoopWithIoDetectorfrom .ownership_risk import OwnershipRiskDetector+from .pandas_iterrows_in_loop import PandasIterrowsInLoopDetectorfrom .pd_concat_in_loop import PdConcatInLoopDetectorfrom .primitive_obsession import PrimitiveObsessionDetectorfrom .prior_defect import PriorDefectDetector@@ -99,6 +100,7 @@ _DETECTOR_FACTORIES: list[type[Biomarker]] = [# Phase 7d — language-specific markers (advisory weight; bounded by the cap).ListInsertZeroInLoopDetector, # type: ignore[list-item]PdConcatInLoopDetector, # type: ignore[list-item]+ PandasIterrowsInLoopDetector, # type: ignore[list-item]JsonParseInLoopDetector, # type: ignore[list-item]ArraySpreadInReduceDetector, # type: ignore[list-item]GoroutineInUnboundedLoopDetector, # type: ignore[list-item]diff --git a/packages/core/src/repowise/core/analysis/health/complexity/walker.py b/packages/core/src/repowise/core/analysis/health/complexity/walker.pyindex 638dd4f17..35e9c6a7f 100644--- a/packages/core/src/repowise/core/analysis/health/complexity/walker.py+++ b/packages/core/src/repowise/core/analysis/health/complexity/walker.py@@ -1031,6 +1031,10 @@ _LOOP_STMT_MARKER_KINDS = frozenset(# Markers for a call that is its OWN iteration construct (``.reduce``), a perf# smell at any loop depth — emitted via ``dialect.bare_call_marker``._BARE_CALL_MARKER_KINDS = frozenset({"array_spread_in_reduce"})+# Markers fired on a loop whose ITERABLE is itself a slow call (``df.iterrows()``)+# — the call lives in the loop header, so the body markers miss it. Emitted via+# ``dialect.loop_iterable_call_marker`` against the loop node.+_LOOP_ITERABLE_CALL_MARKER_KINDS = frozenset({"pandas_iterrows_in_loop"})# Boundary kinds that are *unambiguously blocking* when called synchronously, so# a loop_depth-0 occurrence in a hot function is a ``hot_path_sync_io`` candidate.@@ -1112,6 +1116,7 @@ def _collect_perf_hits(do_loop_call_marker = bool(markers & _LOOP_CALL_MARKER_KINDS)do_loop_stmt_marker = bool(markers & _LOOP_STMT_MARKER_KINDS)do_bare_call_marker = bool(markers & _BARE_CALL_MARKER_KINDS)+ do_loop_iterable_call_marker = bool(markers & _LOOP_ITERABLE_CALL_MARKER_KINDS)do_serial_await = "serial_await_in_loop" in markers# Phase 7b markers.do_nested_io = "nested_loop_with_io" in markers@@ -1193,6 +1198,17 @@ def _collect_perf_hits(if misc[0] == 0:misc[0] = node.start_point[0] + 1+ # A loop whose ITERABLE is itself a slow call (``for _, r in+ # df.iterrows()``): the call sits in the loop header (runs once), so the+ # body call markers never see it. Fire on the loop node itself, at any+ # nesting depth — ``iterrows`` is O(n)-boxing slow on its own.+ if is_loop and do_loop_iterable_call_marker:+ icm = dialect.loop_iterable_call_marker(node)+ if icm is not None:+ hits.append(+ PerfHit(icm, node.start_point[0] + 1, next_func, "", func_start=next_start)+ )+if t in call_kinds:method = dialect.callee_method_name(node) or ""root_name = dialect.callee_root_name(node) or ""diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/base.pyindex 453ee8580..e1cfd6b47 100644--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/base.py@@ -293,6 +293,19 @@ class BasePerfDialect:"""return None+ def loop_iterable_call_marker(self, node: Node) -> str | None:+ """Marker kind for a loop whose *iterable expression is itself a call*.++ Used for ``pandas_iterrows_in_loop`` (``for _, row in df.iterrows()``):+ the offending call sits in the loop HEADER, so it runs once and the body+ :meth:`loop_call_marker` (loop_depth >= 1) never sees it. This hook is+ handed the loop node itself, so the dialect inspects the iterable and+ fires regardless of nesting depth (``iterrows`` is O(n)-boxing slow on+ its own, not only when nested). Default ``None`` so a language that does+ not override it is byte-for-byte unchanged.+ """+ return None+def loop_stmt_marker(self, node: Node, list_names: frozenset[str]) -> str | None:"""Marker kind for a loop-nested *non-call statement* node.diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.pyindex 8df65f05c..a7ceb0101 100644--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py@@ -111,6 +111,8 @@ class PythonPerfDialect(BasePerfDialect):# Phase 7d — Python-specific quadratic anti-patterns."list_insert_zero_in_loop","pd_concat_in_loop",+ # Header-call marker: the slow iterable IS the loop driver.+ "pandas_iterrows_in_loop",})@@ -325,6 +327,27 @@ class PythonPerfDialect(BasePerfDialect):return "pd_concat_in_loop"return None+ def loop_iterable_call_marker(self, node: Node) -> str | None:+ """``for _, row in df.iterrows():`` — row-by-row DataFrame iteration.++ ``DataFrame.iterrows()`` boxes every row into a fresh Series, an order of+ magnitude slower than a vectorized operation (and slower than+ ``itertuples`` / ``to_dict`` when row access is unavoidable). The call+ sits in the loop header, so the body ``loop_call_marker`` misses it.+ Gated to the distinctive method name on a member-access receiver+ (``x.iterrows()``, never a bare ``iterrows(...)``) — high-precision by+ construction, like the ``pd``/``pandas`` ``concat`` root: ``iterrows`` is+ a pandas-only verb with no common collision.+ """+ if node.type != "for_statement":+ return None+ right = node.child_by_field_name("right")+ if right is None or right.type != "call":+ return None+ if self.callee_method_name(right) == "iterrows" and self.callee_is_attribute(right):+ return "pandas_iterrows_in_loop"+ return None+def _receiver_reset_in_loop(self, node: Node, name: str) -> bool:"""True if the call's receiver ``name`` is re-assigned (reset to a freshlist) inside an enclosing loop body — so the per-front-insert is boundeddiff --git a/packages/core/src/repowise/core/analysis/health/scoring.py b/packages/core/src/repowise/core/analysis/health/scoring.pyindex 6fcccdaf6..104883a00 100644--- a/packages/core/src/repowise/core/analysis/health/scoring.py+++ b/packages/core/src/repowise/core/analysis/health/scoring.py@@ -225,6 +225,7 @@ _BIOMARKER_DIMENSIONS: dict[str, set[str]] = {# Phase 7d language-specific markers - performance-only."list_insert_zero_in_loop": {"performance"},"pd_concat_in_loop": {"performance"},+ "pandas_iterrows_in_loop": {"performance"},"json_parse_in_loop": {"performance"},"array_spread_in_reduce": {"performance"},"goroutine_in_unbounded_loop": {"performance"},@@ -324,6 +325,10 @@ _PERFORMANCE_WEIGHT_MULTIPLIER: dict[str, float] = {# precision json_parse / goroutine-spawn ones."list_insert_zero_in_loop": 0.6,"pd_concat_in_loop": 0.6,+ # ``iterrows`` is a documented pandas anti-pattern, high-precision by the+ # distinctive method name; advisory pending corpus volume (no pandas in the+ # OSS gate corpus — by-construction, like pd_concat).+ "pandas_iterrows_in_loop": 0.6,"array_spread_in_reduce": 0.5,"json_parse_in_loop": 0.4,"goroutine_in_unbounded_loop": 0.4,@@ -345,6 +350,7 @@ _PERFORMANCE_CATEGORY: dict[str, str] = {"blocking_io_under_lock": "performance","list_insert_zero_in_loop": "performance","pd_concat_in_loop": "performance",+ "pandas_iterrows_in_loop": "performance","json_parse_in_loop": "performance","array_spread_in_reduce": "performance","goroutine_in_unbounded_loop": "performance",@@ -371,6 +377,7 @@ _PERFORMANCE_HOME: frozenset[str] = frozenset("blocking_io_under_lock","list_insert_zero_in_loop","pd_concat_in_loop",+ "pandas_iterrows_in_loop","json_parse_in_loop","array_spread_in_reduce","goroutine_in_unbounded_loop",diff --git a/tests/unit/health/test_perf_dialects.py b/tests/unit/health/test_perf_dialects.pyindex e64a98f9e..bba556f19 100644--- a/tests/unit/health/test_perf_dialects.py+++ b/tests/unit/health/test_perf_dialects.py@@ -97,7 +97,7 @@ def test_java_cases(src, expected, note):def test_go_fixture_counts():fc = _walk("go/perf_io_in_loop.go", "go")counts = _kinds(fc.perf_hits)- # db (db.Query) · network (http.Get) · filesystem (os.Open ×2, incl. the+ # db (db.Query) · network (http.Get) · filesystem (os.Open x2, incl. the# one in deferInLoop). The constant-bound and out-of-loop queries do not.assert counts["io_in_loop"] == 4assert {h.detail for h in fc.perf_hits if h.kind == "io_in_loop"} == {@@ -382,6 +382,19 @@ def test_python_pd_concat_in_loop():assert ("pd_concat_in_loop", "") in _hits("python", src)+def test_python_pandas_iterrows_in_loop():+ # The iterrows() call lives in the loop HEADER, so the body call-markers+ # never see it — the loop_iterable_call_marker hook fires on the loop node.+ iterrows = "def f(df):\n for _, row in df.iterrows():\n use(row)\n"+ assert ("pandas_iterrows_in_loop", "") in _hits("python", iterrows)+ # itertuples is the recommended faster alternative — never flagged.+ tuples = "def f(df):\n for row in df.itertuples():\n use(row)\n"+ assert not any(k == "pandas_iterrows_in_loop" for k, _ in _hits("python", tuples))+ # A plain collection iterable is not a header-call smell.+ plain = "def f(rows):\n for row in rows:\n use(row)\n"+ assert not any(k == "pandas_iterrows_in_loop" for k, _ in _hits("python", plain))++def test_ts_json_parse_in_loop():src = "function f(xs){ for (const x of xs) { const c = JSON.parse(JSON.stringify(x)); } }"assert ("json_parse_in_loop", "") in _hits("typescript", src)