Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(perf): one plan step per loop and variable for the buffer and membership fixes (#3443)
Yang Baic3bb99d0ebe1
diff --git a/docs/architecture/code-health.md b/docs/architecture/code-health.mdindex 7f3ae5566..f704284fd 100644--- a/docs/architecture/code-health.md+++ b/docs/architecture/code-health.md@@ -746,6 +746,11 @@ prerequisite. A loop already walking chunks is `loop_already_chunked`. Whencall site and loop, however many markers saw it, each naming the call it repeats(`details.sink_call`, or the helper a cross-function path enters). A step that addsa bulk form to the sink comes first only when every site reaches that one sink.+Building a buffer or a set is one edit however many lines of the loop grow the+string or probe the list, so `buffer_string_accumulation` and+`replace_membership_collection` list one step per loop and variable, on the loop's+line, naming the variable (`details.subject`) where the walk could read it as a+plain name, and effort counts those steps.Plans also list validation tests (coverage, call graph, import graph, then`via: "name-match"`). An opportunity links to a `performance_fix` refactoringplan only by exact `opportunity_id`.diff --git a/packages/core/src/repowise/core/analysis/health/complexity/models.py b/packages/core/src/repowise/core/analysis/health/complexity/models.pyindex e9873a38d..03e352696 100644--- a/packages/core/src/repowise/core/analysis/health/complexity/models.py+++ b/packages/core/src/repowise/core/analysis/health/complexity/models.py@@ -343,9 +343,15 @@ class PerfHit:# The per-iteration call as written at ``line`` (``session.get``), so a plan# can name what to batch; empty where the detector kept no call text.sink_call: str = ""+ # The variable the hit is about, as written: the string a ``+=`` grows+ # (``string_concat_in_loop``), the list a membership test probes+ # (``membership_test_against_list_in_loop``). One loop's hits on one+ # variable are one edit, and the plan names it. Empty for every other+ # kind and for a target that is not a plain name (``buf[0] += ...``).+ subject: str = ""def loop_facts(self) -> dict[str, Any]:- """Loop facts and the call text for ``details``.+ """Loop facts, the call text and the subject for ``details``.Each key is absent when unset, so old findings are unchanged."""@@ -354,6 +360,8 @@ class PerfHit:facts["loop_line"] = self.loop_lineif self.sink_call:facts["sink_call"] = self.sink_call+ if self.subject:+ facts["subject"] = self.subjectreturn factsdiff --git a/packages/core/src/repowise/core/analysis/health/complexity/perf_walk.py b/packages/core/src/repowise/core/analysis/health/complexity/perf_walk.pyindex 126ae0ef6..7fba6133e 100644--- a/packages/core/src/repowise/core/analysis/health/complexity/perf_walk.py+++ b/packages/core/src/repowise/core/analysis/health/complexity/perf_walk.py@@ -30,6 +30,7 @@ from .ast_utils import (_dart_signature_sibling,_find_function_entry_name,_find_name,+ _node_text,is_function_node,)from .languages import LanguageNodeMap@@ -365,6 +366,23 @@ def _strip_call_args(text: str) -> str:return "".join(kept)+_MEMBERSHIP = "membership_test_against_list_in_loop"+# A name a plan step can quote: ``out``, ``self.buf``, ``@buf``, ``$acc``.+_SUBJECT_NAME = re.compile(r"[@$]{0,2}[A-Za-z_][\w$]*(?:\.[A-Za-z_][\w$]*)*")+++def _subject_text(node: Node | None) -> str:+ """*node* as written when it is a plain name or member path, else empty."""+ text = "".join(_node_text(node).split()) if node is not None else ""+ return text if len(text) <= _SINK_CALL_MAX and _SUBJECT_NAME.fullmatch(text) else ""+++def _concat_target(node: Node) -> str:+ """The string a ``+=`` statement grows: its left side."""+ left = node.child_by_field_name("left")+ return _subject_text(left if left is not None else next(iter(node.named_children), None))++def _quotable(callee: str) -> bool:return bool(callee) and not callee.startswith("(") and len(callee) <= _SINK_CALL_MAX@@ -759,6 +777,8 @@ def _collect_perf_hits(PerfHit(marker, line, next_func, "", func_start=next_start,loop=loop_facts(call_node, sink=False),+ # ``arr.includes(x)``: the list is the receiver.+ subject=root_name if marker == _MEMBERSHIP else "",))elif method:@@ -811,6 +831,7 @@ def _collect_perf_hits("",func_start=next_start,loop=loop_facts(node, sink=False),+ subject=_concat_target(node),))elif do_loop_stmt_marker:@@ -828,6 +849,12 @@ def _collect_perf_hits("",func_start=next_start,loop=loop_facts(node, sink=False),+ # ``x in big``: the list is the last operand.+ subject=(+ _subject_text(next(iter(reversed(node.named_children)), None))+ if sm == _MEMBERSHIP+ else ""+ ),))diff --git a/packages/core/src/repowise/core/analysis/health/perf/facts.py b/packages/core/src/repowise/core/analysis/health/perf/facts.pyindex f8a800fba..28349c4b2 100644--- a/packages/core/src/repowise/core/analysis/health/perf/facts.py+++ b/packages/core/src/repowise/core/analysis/health/perf/facts.py@@ -154,6 +154,7 @@ def evidence_row(facts: ObservationFacts) -> dict[str, Any]:# walk stored none, so older evidence reads unchanged.**({"loop_line": loop} if (loop := facts.details.get("loop_line")) else {}),**({"sink_call": call} if (call := facts.details.get("sink_call")) else {}),+ **({"subject": subject} if (subject := facts.details.get("subject")) else {}),}diff --git a/packages/core/src/repowise/core/analysis/health/refactoring/performance_fix.py b/packages/core/src/repowise/core/analysis/health/refactoring/performance_fix.pyindex f7b96068e..f4a2e755d 100644--- a/packages/core/src/repowise/core/analysis/health/refactoring/performance_fix.py+++ b/packages/core/src/repowise/core/analysis/health/refactoring/performance_fix.py@@ -34,6 +34,12 @@ _STEPS: dict[str, tuple[str | None, str]] = {"Select one row per key in the query instead of reducing all rows here",),}+# Strategies whose edit is one per loop and variable, however many lines of+# the loop the markers fired on, with the step that names the variable.+_PER_LOOP_STEPS: dict[str, str] = {+ "replace_membership_collection": "Build a set from `{name}` once before the loop and probe it",+ "buffer_string_accumulation": "Collect `{name}` in a list and join it once after the loop",+}# When the fix names the construct it uses, the step says it instead._API_STEPS: dict[str, tuple[str | None, str]] = {"batch_or_prefetch_io": (@@ -74,8 +80,16 @@ def fix_steps(*sink* is the sink every site shares, or ``None``: a bulk form added to onemember's sink would leave the other sites' calls per-key."""+ steps = _intro_steps(strategy, intervention, sink, api)+ steps += _site_steps(strategy, safety, locations, api)+ return [{"order": index, **step} for index, step in enumerate(steps, 1)]+++def _intro_steps(+ strategy: str, intervention: str | None, sink: str | None, api: str | None+) -> list[dict[str, object]]:+ """The step that comes before the sites', where the strategy has one."""intro, per_site = (_API_STEPS if api else _STEPS).get(strategy, (None, ""))- applicability: Applicability = "mechanical" if safety == "proven" else "judgment"steps: list[dict[str, object]] = []target = intervention or sink# With nothing to name, a site step says the whole edit; a strategy with no@@ -90,19 +104,61 @@ def fix_steps("applicability": "judgment",})- if per_site:- steps.extend(+ return steps+++def _site_steps(+ strategy: str, safety: str, locations: list[dict[str, object]], api: str | None+) -> list[dict[str, object]]:+ """One step per place that changes: a call site, or a loop and variable."""+ _, per_site = (_API_STEPS if api else _STEPS).get(strategy, (None, ""))+ if not per_site:+ return []+ applicability: Applicability = "mechanical" if safety == "proven" else "judgment"+ named = _PER_LOOP_STEPS.get(strategy)+ if named and not api:+ return [{- "action": per_site.format(api=api, call=_call_phrase(location.get("call"))),- "symbol": location.get("function_name"),- "file_path": location.get("file_path"),- "line": location.get("line_start"),- **({"loop_line": location["loop_line"]} if location.get("loop_line") else {}),+ "action": named.format(name=edit["subject"]) if edit.get("subject") else per_site,+ "symbol": edit.get("function_name"),+ "file_path": edit.get("file_path"),+ "line": edit.get("loop_line") or edit.get("line_start"),+ **({"loop_line": edit["loop_line"]} if edit.get("loop_line") else {}),"applicability": applicability,}- for location in locations+ for edit in _loop_edits(locations)+ ]+ return [+ {+ "action": per_site.format(api=api, call=_call_phrase(location.get("call"))),+ "symbol": location.get("function_name"),+ "file_path": location.get("file_path"),+ "line": location.get("line_start"),+ **({"loop_line": location["loop_line"]} if location.get("loop_line") else {}),+ "applicability": applicability,+ }+ for location in locations+ ]+++def _loop_edits(locations: list[dict[str, object]]) -> list[dict[str, object]]:+ """The first location of each loop and variable.++ Eight ``out +=`` lines in one loop are one edit, and so are two probes of+ one list; a second variable in the same loop is another. A location with+ no loop line (evidence stored before the walk kept one) stays an edit of+ its own, since nothing says which loop it shares.+ """+ edits: dict[tuple[object, ...], dict[str, object]] = {}+ for location in locations:+ loop = location.get("loop_line")+ key = (+ (location.get("file_path"), loop, location.get("subject"))+ if loop+ else (location.get("file_path"), None, location.get("line_start")))- return [{"order": index, **step} for index, step in enumerate(steps, 1)]+ edits.setdefault(key, location)+ return list(edits.values())def _call_phrase(call: object) -> str:@@ -171,6 +227,8 @@ def _site_locations(evidence: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:)if "call" not in site and (call := _site_call(item)):site["call"] = call+ if "subject" not in site and item.get("subject"):+ site["subject"] = item["subject"]return list(sites.values())@@ -210,14 +268,18 @@ def _suggestion(# sink the loop reaches. The loop's own function is where the keys are# collected, never what gains the bulk form.batched_on_helper = opportunity.intervention_kind == "shared_helper"- steps = fix_steps(+ intro = _intro_steps(fix.strategy,- fix.safety,intervention if fix.strategy != "batch_or_prefetch_io" or batched_on_helper else None,_shared_sink(opportunity),- locations,fix.api,)+ edits = _site_steps(fix.strategy, fix.safety, locations, fix.api)+ steps = [{"order": index, **step} for index, step in enumerate(intro + edits, 1)]+ # One edit per site, as before, less the sites a loop's one step covers.+ # Sites past the evidence cap stay one each: nothing says which share a loop.+ per_site = len(locations) if edits else 0+ sites = opportunity.affected_call_sites_total - (per_site - len(edits))mechanical = sum(step["applicability"] == "mechanical" for step in steps)return RefactoringSuggestion(refactoring_type="performance_fix",@@ -256,9 +318,7 @@ def _suggestion(# Performance is a separate score dimension and its findings# intentionally carry zero defect-health impact.impact_delta=0.0,- effort_bucket=edit_effort(- opportunity.affected_call_sites_total + (len(steps) > len(locations))- ),+ effort_bucket=edit_effort(sites + (len(intro) + per_site > len(locations))),blast_radius={"files": sorted({item["file_path"] for item in opportunity.evidence}),"file_count": opportunity.affected_files_total,diff --git a/tests/unit/health/test_perf_plan_steps_per_loop.py b/tests/unit/health/test_perf_plan_steps_per_loop.pynew file mode 100644index 000000000..8874246ce--- /dev/null+++ b/tests/unit/health/test_perf_plan_steps_per_loop.py@@ -0,0 +1,359 @@+"""A buffer or a set is built once per loop, so its plan is one step per loop.++``buffer_string_accumulation`` and ``replace_membership_collection`` had one+step per source line: a loop with eight ``out +=`` lines gave eight identical+steps and an effort of eight edits (#3268). The edit is one per loop and+variable, and the step names the variable when the walk kept it.+"""++from __future__ import annotations++import pytest++from repowise.core.analysis.health.complexity import walk_file+from repowise.core.analysis.health.perf.opportunities import build_performance_opportunities+from repowise.core.analysis.health.refactoring.performance_fix import (+ _site_locations,+ fix_steps,+ performance_fix_suggestions,+)+from tests.unit.health.test_perf_opportunities import _finding++_BUFFER = "buffer_string_accumulation"+_MEMBERSHIP = "replace_membership_collection"+_CONCAT = "string_concat_in_loop"+_PROBE = "membership_test_against_list_in_loop"+++def _evidence(lines, *, loop_line=5, subject=None, path="a.py"):+ return [+ {+ "file_path": path,+ "function_name": "f",+ "line_start": line,+ "line_end": line,+ **({"loop_line": loop_line} if loop_line else {}),+ **({"subject": subject} if subject else {}),+ }+ for line in lines+ ]+++def _steps(strategy, evidence, safety="judgment"):+ return fix_steps(strategy, safety, None, None, _site_locations(evidence))+++def _plan(rows, **options):+ opportunities = build_performance_opportunities(rows, **options)+ assert len(opportunities) == 1+ (suggestion,) = performance_fix_suggestions(opportunities)+ return suggestion+++# -- the steps ------------------------------------------------------------------+++ ("strategy", "wording"),+ [+ (_BUFFER, "Accumulate into a list and join once after the loop"),+ (_MEMBERSHIP, "Build a set once before the loop and probe it"),+ ],+)+def test_the_report_s_eight_lines_in_one_loop_are_one_step(strategy, wording):+ steps = _steps(strategy, _evidence(range(10, 18)))++ assert [(s["order"], s["action"], s["line"], s["loop_line"]) for s in steps] == [+ (1, wording, 5, 5)+ ]+++ ("strategy", "name", "action"),+ [+ (_BUFFER, "out", "Collect `out` in a list and join it once after the loop"),+ (_BUFFER, "self.buf", "Collect `self.buf` in a list and join it once after the loop"),+ (_MEMBERSHIP, "seen", "Build a set from `seen` once before the loop and probe it"),+ ],+)+def test_the_step_names_the_variable(strategy, name, action):+ steps = _steps(strategy, _evidence(range(10, 14), subject=name))++ assert [s["action"] for s in steps] == [action]+++def test_two_variables_in_one_loop_are_two_steps():+ evidence = (+ _evidence([10, 12], subject="out")+ + _evidence([11, 13], subject="log")+ + _evidence([14], subject="out")+ )++ steps = _steps(_BUFFER, evidence)++ assert [(s["order"], s["action"], s["line"]) for s in steps] == [+ (1, "Collect `out` in a list and join it once after the loop", 5),+ (2, "Collect `log` in a list and join it once after the loop", 5),+ ]+++def test_two_loops_in_one_function_are_two_steps():+ evidence = _evidence([10, 11], subject="out") + _evidence(+ [22, 23, 24], loop_line=20, subject="out"+ )++ steps = _steps(_BUFFER, evidence)++ assert [(s["line"], s["loop_line"], s["symbol"]) for s in steps] == [(5, 5, "f"), (20, 20, "f")]+++def test_the_same_loop_line_in_another_file_is_another_loop():+ evidence = _evidence([10, 11], subject="out") + _evidence([10], subject="out", path="b.py")++ assert [(s["file_path"], s["line"]) for s in _steps(_BUFFER, evidence)] == [+ ("a.py", 5),+ ("b.py", 5),+ ]+++def test_sites_with_no_name_share_one_step_with_today_s_wording():+ """Beside a named variable in the same loop, they are their own edit."""+ evidence = _evidence([10, 11], subject="out") + _evidence([12, 13])++ assert [s["action"] for s in _steps(_BUFFER, evidence)] == [+ "Collect `out` in a list and join it once after the loop",+ "Accumulate into a list and join once after the loop",+ ]+++def test_evidence_with_no_loop_line_stays_one_step_per_site():+ """Stored before the walk kept the loop: nothing says which loop a site is in."""+ steps = _steps(_BUFFER, _evidence([10, 11, 12], loop_line=None, subject="out"))++ assert [s["line"] for s in steps] == [10, 11, 12]+ assert all("loop_line" not in s for s in steps)+ assert {s["action"] for s in steps} == {+ "Collect `out` in a list and join it once after the loop"+ }+++def test_a_proven_plan_s_loop_step_is_mechanical():+ assert [s["applicability"] for s in _steps(_BUFFER, _evidence([10, 11]), "proven")] == [+ "mechanical"+ ]+ assert [s["applicability"] for s in _steps(_BUFFER, _evidence([10, 11]))] == ["judgment"]+++def test_the_other_strategies_keep_one_step_per_site():+ evidence = [{**item, "sink_call": "db.get"} for item in _evidence([10, 11, 12], subject="x")]++ steps = fix_steps("batch_or_prefetch_io", "judgment", None, None, _site_locations(evidence))++ assert [(s["line"], s["loop_line"]) for s in steps] == [(10, 5), (11, 5), (12, 5)]+ assert all("db.get" in s["action"] for s in steps)+++def test_the_locations_still_list_every_line_and_carry_the_name():+ locations = _site_locations(_evidence([10, 11], subject="out"))++ assert [(loc["line_start"], loc["loop_line"], loc["subject"]) for loc in locations] == [+ (10, 5, "out"),+ (11, 5, "out"),+ ]+ assert "subject" not in _site_locations(_evidence([10]))[0]+++# -- from a finding to the plan ---------------------------------------------------+++def test_eight_lines_in_one_loop_are_one_edit_of_effort():+ rows = [+ _finding("a.py", line, marker=_CONCAT, boundary="", loop_line=5, subject="out")+ for line in range(10, 18)+ ]++ plan = _plan(rows)++ assert plan.plan["strategy"] == _BUFFER+ assert [s["action"] for s in plan.plan["steps"]] == [+ "Collect `out` in a list and join it once after the loop"+ ]+ assert len(plan.plan["affected_locations"]) == 8+ assert plan.effort_bucket == "S"+++def test_effort_counts_loops_and_variables():+ rows = (+ [_finding("a.py", n, marker=_CONCAT, boundary="", loop_line=5, subject="out") for n in (6, 7)]+ + [_finding("a.py", n, marker=_CONCAT, boundary="", loop_line=5, subject="log") for n in (8, 9)]+ + [_finding("a.py", n, marker=_CONCAT, boundary="", loop_line=20, subject="out") for n in (21, 22)]+ )++ plan = _plan(rows)++ assert len(plan.plan["steps"]) == 3+ assert plan.effort_bucket == "M"+++def test_sites_past_the_evidence_cap_still_count_one_each():+ """Six lines of one loop, four of them shown: the four are one edit, and+ nothing says the other two are in that loop."""+ rows = [+ _finding("a.py", line, marker=_CONCAT, boundary="", loop_line=5, subject="out")+ for line in range(10, 16)+ ]++ plan = _plan(rows, evidence_limit=4)++ assert len(plan.plan["affected_locations"]) == 4+ assert len(plan.plan["steps"]) == 1+ assert plan.effort_bucket == "M"+++def test_a_per_site_strategy_s_effort_is_what_it_was():+ """Three call sites of one helper are three site steps and the bulk form:+ four edits, ``L``. Three alone would be ``M``."""+ sink = ("shared.py::load", "git.py::head")+ rows = [+ _finding("a.py", line, loop_line=8, call_path=("a.py::run", *sink)) for line in (10, 11, 12)+ ]++ plan = _plan(rows)++ assert len(plan.plan["steps"]) == 4+ assert plan.effort_bucket == "L"+++# -- the walk keeps the name ------------------------------------------------------+++def _subjects(path, language, source):+ return [+ (hit.kind, hit.line, hit.subject)+ for hit in walk_file(path, language, source.encode()).perf_hits+ ]+++def test_python_records_the_string_and_the_list():+ source = (+ "def f(rows):\n"+ " out = ''\n"+ " big = [1, 2, 3]\n"+ " for r in rows:\n"+ " out += 'a'\n"+ " self.buf += 'b'\n"+ " if r in big:\n"+ " out += 'c'\n"+ " if r not in big:\n"+ " parts[0] += 'd'\n"+ " return out\n"+ )++ assert _subjects("a.py", "python", source) == [+ (_CONCAT, 5, "out"),+ (_CONCAT, 6, "self.buf"),+ (_PROBE, 7, "big"),+ (_CONCAT, 8, "out"),+ (_PROBE, 9, "big"),+ # A subscript is not a name a step can quote.+ (_CONCAT, 10, ""),+ ]+++def test_typescript_records_the_string_and_the_array():+ source = (+ "export function f(rows: string[]) {\n"+ " let out = '';\n"+ " const big = [1, 2, 3];\n"+ " for (const r of rows) {\n"+ " out += 'a';\n"+ " this.buf += 'b';\n"+ " if (big.includes(r)) { out += 'c'; }\n"+ " }\n"+ " return out;\n"+ "}\n"+ )++ assert _subjects("a.ts", "typescript", source) == [+ (_CONCAT, 5, "out"),+ (_CONCAT, 6, "this.buf"),+ (_PROBE, 7, "big"),+ (_CONCAT, 7, "out"),+ ]+++ ("path", "language", "source", "expected"),+ [+ pytest.param(+ "a.go",+ "go",+ 'package m\nfunc f(rows []string) string {\n\tout := ""\n'+ '\tfor range rows {\n\t\tout += "a"\n\t}\n\treturn out\n}\n',+ [(_CONCAT, 5, "out")],+ id="go",+ ),+ pytest.param(+ "a.rb",+ "ruby",+ "def f(rows)\n out = ''\n rows.each do |r|\n out += 'a'\n @buf += 'b'\n end\n out\nend\n",+ [(_CONCAT, 4, "out"), (_CONCAT, 5, "@buf")],+ id="ruby",+ ),+ pytest.param(+ "a.kt",+ "kotlin",+ 'fun f(rows: List<String>): String {\n var out = ""\n for (r in rows) {\n'+ ' out += "a"\n }\n return out\n}\n',+ [(_CONCAT, 4, "out")],+ id="kotlin",+ ),+ pytest.param(+ "a.scala",+ "scala",+ 'object O {\n def f(rows: List[String]): String = {\n var out = ""\n'+ ' for (r <- rows) {\n out += "a"\n }\n out\n }\n}\n',+ [(_CONCAT, 5, "out")],+ id="scala",+ ),+ pytest.param(+ "a.dart",+ "dart",+ "String f(List<String> rows) {\n var out = '';\n for (final r in rows) {\n"+ " out += 'a';\n }\n return out;\n}\n",+ [(_CONCAT, 4, "out")],+ id="dart",+ ),+ ],+)+def test_every_dialect_with_the_marker_records_the_string(path, language, source, expected):+ hits = _subjects(path, language, source)+ if not hits:+ pytest.skip(f"tree-sitter language pack missing for {language}")+ assert hits == expected+++def test_no_other_kind_of_hit_has_a_subject():+ """A sink, and a call marker that is not the membership one."""+ source = (+ "import requests\n"+ "def f(urls, items):\n"+ " for u in urls:\n"+ " requests.get(u)\n"+ " items.insert(0, u)\n"+ )++ hits = walk_file("a.py", "python", source.encode()).perf_hits++ assert [hit.kind for hit in hits] == ["io_in_loop", "list_insert_zero_in_loop"]+ assert all(hit.subject == "" for hit in hits)+ assert all("subject" not in hit.loop_facts() for hit in hits)+++def test_the_name_travels_in_the_finding_s_details():+ source = "def f(rows):\n out = ''\n for r in rows:\n out += 'a'\n return out\n"++ (hit,) = walk_file("a.py", "python", source.encode()).perf_hits++ assert hit.loop_facts()["subject"] == "out"+ assert hit.loop_facts()["loop_line"] == 3