Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(health): chunk the whole-repository target sets the queue finalize reads (#4403)

Ei-Aywb96f3a795a76
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/coverage_map.py b/packages/core/src/repowise/core/persistence/crud/analysis/coverage_map.py
index 9d3f309b8..a0e087a18 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/coverage_map.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/coverage_map.py
@@ -20,7 +20,8 @@ from typing import Any
from sqlalchemy import delete, insert, select
from sqlalchemy.ext.asyncio import AsyncSession
-from ...models import TestCoverageEntry, _new_uuid, _now_utc
+from ...models import TestCoverageEntry, _new_uuid
+from .._shared import _BATCH_SIZE, _now_utc
# Upper bound on rows persisted per repo, so a pathological O(tests x files)
# report can't balloon the table unbounded. Generous headroom: a 2000-test
@@ -207,8 +208,9 @@ async def tests_covering_files(
) -> dict[str, list[dict[str, Any]]]:
"""Bulk reverse index for recommendation validation.
- Unlike :func:`tests_covering`, this performs one indexed query for the
- complete target set. Callers may then apply plan-specific line ranges in
+ Unlike :func:`tests_covering`, this performs indexed queries over the
+ complete target set, one per bounded piece of it. Callers may then apply
+ plan-specific line ranges in
memory without turning recommendation hydration into one query per plan.
Rows and buckets are ordered deterministically; ``covered_lines`` remains
complete so a caller can distinguish a measured line hit from a file-only
@@ -216,20 +218,28 @@ async def tests_covering_files(
"""
if not source_files:
return {}
- result = await session.execute(
- select(TestCoverageEntry)
- .where(
- TestCoverageEntry.repository_id == repository_id,
- TestCoverageEntry.source_file.in_(sorted(source_files)),
- )
- .order_by(
- TestCoverageEntry.source_file.asc(),
- TestCoverageEntry.test_id.asc(),
- TestCoverageEntry.test_file.asc(),
- )
- )
out: dict[str, list[dict[str, Any]]] = {}
- for row in result.scalars().all():
+ # A full re-score asks for every file in the repository, and SQLite caps the
+ # bound parameters per statement, so the set goes out in fixed-size pieces
+ # (the same reason every other ``IN`` list here does). A file appears in
+ # exactly one piece, so each file's rows keep their order.
+ ordered = sorted(source_files)
+ rows: list[TestCoverageEntry] = []
+ for start in range(0, len(ordered), _BATCH_SIZE):
+ result = await session.execute(
+ select(TestCoverageEntry)
+ .where(
+ TestCoverageEntry.repository_id == repository_id,
+ TestCoverageEntry.source_file.in_(ordered[start : start + _BATCH_SIZE]),
+ )
+ .order_by(
+ TestCoverageEntry.source_file.asc(),
+ TestCoverageEntry.test_id.asc(),
+ TestCoverageEntry.test_file.asc(),
+ )
+ )
+ rows.extend(result.scalars().all())
+ for row in rows:
out.setdefault(row.source_file, []).append(
{
"test_id": row.test_id,
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/fix_first.py b/packages/core/src/repowise/core/persistence/crud/analysis/fix_first.py
index 9ac4feac4..4adce9720 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/fix_first.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/fix_first.py
@@ -60,6 +60,7 @@ from repowise.core.analysis.health.rows import detail_map
from repowise.core.analysis.health.scoring import history_biomarkers
from repowise.core.ingestion.git_indexer.function_blame import _MIN_COMMITS_FOR_BLAME
+from ...batches import chunked
from ...models import (
DeadCodeFinding,
GitFunctionBlame,
@@ -84,13 +85,25 @@ from .refactoring_recommendations import repository_checkout, validation_inputs
FINDING_FILES = 100
+async def _rows_in(
+ session: AsyncSession, items: Iterable[Any], statement: Callable[[Any], Any]
+) -> list[Any]:
+ """The rows of ``statement(piece)`` for each piece of *items*, concatenated.
+
+ For a statement whose only large bound list is the piece and whose rows no
+ two pieces share; SQLite caps the parameters of one statement.
+ """
+ rows: list[Any] = []
+ for piece in chunked(sorted(set(items))):
+ rows.extend(plain_rows(await session.execute(statement(piece))))
+ return rows
+
+
async def _metrics(session: AsyncSession, repo_id: str, paths: set[str] | None) -> list[Any]:
"""The named files' rows; ``None`` reads every file."""
- if paths is not None and not paths:
- return []
- scoped = HealthFileMetric.file_path.in_(paths) if paths is not None else true()
- return plain_rows(
- await session.execute(
+
+ def query(scoped: Any) -> Any:
+ return (
select(
HealthFileMetric.file_path,
HealthFileMetric.nloc,
@@ -113,6 +126,11 @@ async def _metrics(session: AsyncSession, repo_id: str, paths: set[str] | None)
)
.where(HealthFileMetric.repository_id == repo_id, scoped)
)
+
+ if paths is None:
+ return plain_rows(await session.execute(query(true())))
+ return await _rows_in(
+ session, paths, lambda piece: query(HealthFileMetric.file_path.in_(piece))
)
@@ -168,10 +186,14 @@ async def _function_changes(
keys = {key for _path, key in leads if key}
if not keys:
return {}, set()
- rows = await session.execute(
- select(b.symbol_id, b.mod_count).where(b.repository_id == repo_id, b.symbol_id.in_(keys))
+ rows = await _rows_in(
+ session,
+ keys,
+ lambda piece: select(b.symbol_id, b.mod_count).where(
+ b.repository_id == repo_id, b.symbol_id.in_(piece)
+ ),
)
- changes = dict(rows.all())
+ changes = {r.symbol_id: r.mod_count for r in rows}
matched = {path for path, key in leads if key in changes}
unmatched = {path for path, key in leads if key} - matched
return changes, await _blame_skipped(session, repo_id, unmatched)
@@ -187,24 +209,28 @@ async def _blame_skipped(session: AsyncSession, repo_id: str, paths: set[str]) -
"""
if not paths:
return set()
- floor = await session.execute(
- select(GitMetadata.file_path).where(
+ floor = await _rows_in(
+ session,
+ paths,
+ lambda piece: select(GitMetadata.file_path).where(
GitMetadata.repository_id == repo_id,
- GitMetadata.file_path.in_(paths),
+ GitMetadata.file_path.in_(piece),
GitMetadata.commit_count_total >= _MIN_COMMITS_FOR_BLAME,
- )
+ ),
)
- candidates = set(floor.scalars().all())
+ candidates = {r.file_path for r in floor}
if not candidates:
return set()
b = GitFunctionBlame
# Unindexed, but read only for the few files that pass the floor unmatched.
- blamed = await session.execute(
- select(b.file_path)
- .where(b.repository_id == repo_id, b.file_path.in_(candidates))
- .distinct()
+ blamed = await _rows_in(
+ session,
+ candidates,
+ lambda piece: (
+ select(b.file_path).where(b.repository_id == repo_id, b.file_path.in_(piece)).distinct()
+ ),
)
- return candidates - set(blamed.scalars().all())
+ return candidates - {r.file_path for r in blamed}
async def _basis(session: AsyncSession, repo_id: str) -> dict[str, str | None]:
@@ -279,34 +305,38 @@ async def _findings(
return []
f = HealthFinding
history = history_biomarkers()
- return plain_rows(
- await session.execute(
- select(
- f.file_path,
- f.biomarker_type,
- f.severity,
- f.function_name,
- f.line_start,
- f.line_end,
- f.reason,
- f.health_impact,
- f.public_id,
- f.dimension,
- f.status,
- # History numbers become plain context sentences.
- f.details_json,
- ).where(
- queue_findings(repo_id),
- or_(
- f.file_path.in_(full),
- and_(
- f.file_path.in_(planned - full),
- or_(f.biomarker_type.in_(history), f.function_name.in_(functions)),
- ),
- ),
- )
- )
+ columns = (
+ f.file_path,
+ f.biomarker_type,
+ f.severity,
+ f.function_name,
+ f.line_start,
+ f.line_end,
+ f.reason,
+ f.health_impact,
+ f.public_id,
+ f.dimension,
+ f.status,
+ # History numbers become plain context sentences.
+ f.details_json,
)
+ rows = await _rows_in(
+ session,
+ full,
+ lambda piece: select(*columns).where(queue_findings(repo_id), f.file_path.in_(piece)),
+ )
+ # The function names grow with the plans, so they are matched here rather
+ # than bound beside the paths.
+ quoted = await _rows_in(
+ session,
+ planned - full,
+ lambda piece: select(*columns).where(
+ queue_findings(repo_id),
+ f.file_path.in_(piece),
+ or_(f.biomarker_type.in_(history), f.function_name.is_not(None)),
+ ),
+ )
+ return rows + [r for r in quoted if r.biomarker_type in history or r.function_name in functions]
async def _refactoring(session: AsyncSession, repo_id: str) -> list[Any]:
@@ -434,35 +464,39 @@ async def _plans(
``files=None`` reads the Extract Method plans of every file."""
ids = {s.get("plan_id") for s in steps if s.get("plan_id")}
s = RefactoringSuggestion
- named = s.public_id.in_(ids) if ids else None
extract = and_(s.refactoring_type == "extract_method", s.status == "open")
- extractions = (
- extract if files is None else and_(extract, s.file_path.in_(files)) if files else None
- )
helpers = and_(s.refactoring_type == "extract_helper", s.status == "open")
- wanted = [c for c in (named, extractions, helpers) if c is not None]
- if not wanted:
- return []
- return plain_rows(
- await session.execute(
- select(
- s.public_id,
- s.refactoring_type,
- s.file_path,
- s.target_symbol,
- s.status,
- s.evidence_json,
- s.plan_json,
- s.rank_json,
- ).where(
- s.repository_id == repo_id,
- or_(*wanted),
- # The registry's withheld types, as on every plan list. Fix
- # first is its own measured lens, so no language gate applies.
- shown_plan_predicate(include_unverified=True),
- )
+
+ def query(wanted: Any) -> Any:
+ return select(
+ s.id,
+ s.public_id,
+ s.refactoring_type,
+ s.file_path,
+ s.target_symbol,
+ s.status,
+ s.evidence_json,
+ s.plan_json,
+ s.rank_json,
+ ).where(
+ s.repository_id == repo_id,
+ wanted,
+ # The registry's withheld types, as on every plan list. Fix
+ # first is its own measured lens, so no language gate applies.
+ shown_plan_predicate(include_unverified=True),
)
+
+ # The ids and the files go out in pieces, so a row two of these reads
+ # match is kept once, by its id.
+ rows = plain_rows(
+ await session.execute(query(or_(helpers, extract) if files is None else helpers))
)
+ rows += await _rows_in(session, ids, lambda piece: query(s.public_id.in_(piece)))
+ if files:
+ rows += await _rows_in(
+ session, files, lambda piece: query(and_(extract, s.file_path.in_(piece)))
+ )
+ return list({row.id: row for row in rows}.values())
async def _symbol_lines(session: AsyncSession, repo_id: str, performance: list[Any]) -> dict[str, int]:
@@ -484,12 +518,14 @@ async def _symbol_lines(session: AsyncSession, repo_id: str, performance: list[A
if not wanted:
return {}
g = GraphNode
- rows = await session.execute(
- select(g.node_id, g.start_line).where(
- g.repository_id == repo_id, g.node_id.in_(wanted), g.start_line.is_not(None)
- )
+ rows = await _rows_in(
+ session,
+ wanted,
+ lambda piece: select(g.node_id, g.start_line).where(
+ g.repository_id == repo_id, g.node_id.in_(piece), g.start_line.is_not(None)
+ ),
)
- return {node_id: line for node_id, line in rows.all()}
+ return {r.node_id: r.start_line for r in rows}
def _span_suggestion(path: str, function: Any, start: Any, end: Any) -> PlanSuggestion:
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/function_facts.py b/packages/core/src/repowise/core/persistence/crud/analysis/function_facts.py
index 96d0f5be4..52d3435ba 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/function_facts.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/function_facts.py
@@ -216,22 +216,31 @@ async def tests_for_functions(
async def tests_in_files(
session: AsyncSession, repository_id: str, paths: Iterable[str]
) -> dict[str, tuple[MappedTest, ...]]:
- """:func:`tests_for_functions` for every function in *paths*, in one statement."""
+ """:func:`tests_for_functions` for every function in *paths*, in bounded statements."""
files = sorted(set(paths))
if not files:
return {}
- result = await session.execute(
- select(FunctionFact.symbol_id, FunctionFact.tests_json)
- .join(
- GraphNode,
- and_(
- GraphNode.repository_id == FunctionFact.repository_id,
- GraphNode.node_id == FunctionFact.symbol_id,
- ),
+ out: dict[str, tuple[MappedTest, ...]] = {}
+ # A re-score over the whole repository asks for every file at once, and the
+ # number of bound parameters per statement is capped, so the set goes out in
+ # pieces. Each row belongs to exactly one file, so the result is the same.
+ for start in range(0, len(files), _CHUNK):
+ result = await session.execute(
+ select(FunctionFact.symbol_id, FunctionFact.tests_json)
+ .join(
+ GraphNode,
+ and_(
+ GraphNode.repository_id == FunctionFact.repository_id,
+ GraphNode.node_id == FunctionFact.symbol_id,
+ ),
+ )
+ .where(
+ FunctionFact.repository_id == repository_id,
+ GraphNode.file_path.in_(files[start : start + _CHUNK]),
+ )
)
- .where(FunctionFact.repository_id == repository_id, GraphNode.file_path.in_(files))
- )
- return dict(_decoded(result.all(), None))
+ out.update(_decoded(result.all(), None))
+ return out
def _decoded(
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/health/findings.py b/packages/core/src/repowise/core/persistence/crud/analysis/health/findings.py
index 59b599871..225b64bf6 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/health/findings.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/health/findings.py
@@ -21,6 +21,7 @@ from .....analysis.health.governance import GOVERNANCE_BIOMARKERS
from .....analysis.health.rows import detail_map
from .....analysis.health.scoring import ADVISORY_DIMENSION
from .....analysis.health.worth import LowPriority, finding_priorities
+from ....batches import chunked
from ....models import HealthFinding, _new_uuid, _now_utc
from ..._shared import _BATCH_SIZE
from ...graph import get_test_file_paths
@@ -218,33 +219,34 @@ async def upsert_health_findings(
Used by the incremental ``repowise update`` path so unchanged files
keep their findings instead of being wiped on every partial re-index.
"""
- if not file_paths:
- return
- predicates = [
- HealthFinding.repository_id == repository_id,
- HealthFinding.file_path.in_(file_paths),
- ]
- if dimension is not None:
- predicates.append(HealthFinding.dimension == dimension)
- await _delete_open(session, predicates)
-
- allowed = set(file_paths)
- scoped = [
- finding
- for finding in findings
- if (finding.file_path if hasattr(finding, "file_path") else finding.get("file_path"))
- in allowed
- and (
- dimension is None
- or (
- getattr(finding, "dimension", None)
- if hasattr(finding, "dimension")
- else finding.get("dimension")
+ # One pass per piece of the paths. A finding's public id carries its file,
+ # so it only ever folds into a triaged row of its own piece.
+ for chunk in chunked(sorted(set(file_paths))):
+ predicates = [
+ HealthFinding.repository_id == repository_id,
+ HealthFinding.file_path.in_(chunk),
+ ]
+ if dimension is not None:
+ predicates.append(HealthFinding.dimension == dimension)
+ await _delete_open(session, predicates)
+
+ allowed = set(chunk)
+ scoped = [
+ finding
+ for finding in findings
+ if (finding.file_path if hasattr(finding, "file_path") else finding.get("file_path"))
+ in allowed
+ and (
+ dimension is None
+ or (
+ getattr(finding, "dimension", None)
+ if hasattr(finding, "dimension")
+ else finding.get("dimension")
+ )
+ == dimension
)
- == dimension
- )
- ]
- await _insert_keeping_triage(session, repository_id, scoped, predicates)
+ ]
+ await _insert_keeping_triage(session, repository_id, scoped, predicates)
async def update_health_finding_status(
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/health/metrics.py b/packages/core/src/repowise/core/persistence/crud/analysis/health/metrics.py
index 5b5db5066..a30d1cdab 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/health/metrics.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/health/metrics.py
@@ -20,6 +20,7 @@ from .....analysis.health.ranking import sort_metrics_worst_first
from .....analysis.health.scope import scores_language
from .....analysis.health.scoring import SCORE_FIELDS
from .....test_paths import is_test_related_path
+from ....batches import chunked
from ....models import GraphNode, HealthFileMetric, HealthFinding, _new_uuid
from ..._shared import _BATCH_SIZE, _batch_delete_in
from .exclude import _filter_excluded_paths, health_exclude_spec
@@ -122,13 +123,15 @@ async def upsert_health_metrics(
if not metrics:
return
paths = [m.file_path if hasattr(m, "file_path") else m["file_path"] for m in metrics]
- existing = await session.execute(
- select(HealthFileMetric).where(
- HealthFileMetric.repository_id == repository_id,
- HealthFileMetric.file_path.in_(paths),
+ by_path: dict[str, HealthFileMetric] = {}
+ for chunk in chunked(paths):
+ existing = await session.execute(
+ select(HealthFileMetric).where(
+ HealthFileMetric.repository_id == repository_id,
+ HealthFileMetric.file_path.in_(chunk),
+ )
)
- )
- by_path = {row.file_path: row for row in existing.scalars().all()}
+ by_path.update((row.file_path, row) for row in existing.scalars().all())
for m in metrics:
data = _health_metric_row_data(m) if hasattr(m, "file_path") else dict(m)
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/refactoring.py b/packages/core/src/repowise/core/persistence/crud/analysis/refactoring.py
index ea9b9692b..6ea296704 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/refactoring.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/refactoring.py
@@ -12,7 +12,7 @@ what makes them agree.
from __future__ import annotations
import json
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any
from sqlalchemy import (
@@ -30,6 +30,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm.attributes import set_committed_value
from ....analysis.finding_registry import excluded_types
+from ...batches import chunked
from ...models import RefactoringSuggestion, _new_uuid, _now_utc
from ...sql import rule_predicate
from .._shared import _BATCH_SIZE, _finding_file_path
@@ -127,7 +128,7 @@ def _refactoring_row_kwargs(
def _scope_predicates(
repository_id: str,
*,
- file_paths: list[str] | None,
+ file_paths: Sequence[str] | None,
refactoring_type: str | None,
) -> list[Any]:
predicates: list[Any] = [RefactoringSuggestion.repository_id == repository_id]
@@ -300,21 +301,23 @@ async def finalize_refactoring_suggestions(
)
public_ids = assign_public_ids(scoped)
- stored_rows = list(
- (
- await session.execute(
- select(RefactoringSuggestion).where(
- *_scope_predicates(
- repository_id,
- file_paths=file_paths,
- refactoring_type=refactoring_type,
+ stored_rows: list[RefactoringSuggestion] = []
+ for chunk in [None] if allowed is None else chunked(sorted(allowed)):
+ stored_rows.extend(
+ (
+ await session.execute(
+ select(RefactoringSuggestion).where(
+ *_scope_predicates(
+ repository_id,
+ file_paths=chunk,
+ refactoring_type=refactoring_type,
+ )
)
)
)
+ .scalars()
+ .all()
)
- .scalars()
- .all()
- )
reusable = await _reusable_rows(
session, repository_id, stored_rows, public_ids, REFACTORING_MODEL_VERSION
)
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/refactoring_recommendations.py b/packages/core/src/repowise/core/persistence/crud/analysis/refactoring_recommendations.py
index baa5ecdb2..39a70e35a 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/refactoring_recommendations.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/refactoring_recommendations.py
@@ -55,6 +55,7 @@ from repowise.core.analysis.test_reachability import (
from repowise.core.ingestion.package_roots import scan_package_manifests
from ...models import GraphMetric, GraphNode, HealthFileMetric, Repository
+from .._shared import _BATCH_SIZE
from ..graph import get_graph_metrics
from .coverage_map import tests_covering_files
from .function_facts import tests_in_files
@@ -285,18 +286,25 @@ async def _test_runs(checkout: str | None, test_files: set[str]) -> TestRuns:
async def _symbol_spans(
session: AsyncSession, repository_id: str, files: Sequence[str]
) -> dict[str, list[SymbolSpan]]:
- """Each file's symbol ids with their line spans, in one statement."""
+ """Each file's symbol ids with their line spans, in bounded statements."""
if not files:
return {}
- rows = await session.execute(
- select(GraphNode.file_path, GraphNode.node_id, GraphNode.start_line, GraphNode.end_line)
- .where(GraphNode.repository_id == repository_id)
- .where(GraphNode.node_type == "symbol")
- .where(GraphNode.file_path.in_(list(files)))
- )
spans: dict[str, list[SymbolSpan]] = {}
- for file_path, node_id, start, end in rows:
- spans.setdefault(file_path, []).append((node_id, int(start or 0), int(end or 0)))
+ # The finalize ranks every plan at once, so *files* can be the whole
+ # repository. The statement is issued in fixed-size pieces to stay under the
+ # bound-parameter cap; a file appears in one piece, so its order is kept.
+ paths = sorted(set(files))
+ for start in range(0, len(paths), _BATCH_SIZE):
+ rows = await session.execute(
+ select(GraphNode.file_path, GraphNode.node_id, GraphNode.start_line, GraphNode.end_line)
+ .where(GraphNode.repository_id == repository_id)
+ .where(GraphNode.node_type == "symbol")
+ .where(GraphNode.file_path.in_(paths[start : start + _BATCH_SIZE]))
+ )
+ for file_path, node_id, start_line, end_line in rows:
+ spans.setdefault(file_path, []).append(
+ (node_id, int(start_line or 0), int(end_line or 0))
+ )
return spans
diff --git a/tests/unit/persistence/test_large_target_sets.py b/tests/unit/persistence/test_large_target_sets.py
new file mode 100644
index 000000000..6d60831e8
--- /dev/null
+++ b/tests/unit/persistence/test_large_target_sets.py
@@ -0,0 +1,115 @@
+"""The recommendation reads are asked for every plan's files at once.
+
+Finalizing the queues ranks every live plan in the repository, so the target
+file set of these reads is the size of the repository. SQLite caps the bound
+parameters of one statement (32766 in current builds, 999 before 3.32), so an
+``IN`` list built from that set has to go out in pieces: one statement over
+every path raises ``sqlite3.OperationalError: too many SQL variables`` and the
+finalize, which is shared by the health persist, the queue rebuild after prune
+and the idle-file re-score, fails with it.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.persistence.crud.analysis.coverage_map import (
+ tests_covering_files as covering_files,
+)
+from repowise.core.persistence.crud.analysis.function_facts import (
+ tests_in_files as load_tests_in_files,
+)
+from repowise.core.persistence.crud.analysis.refactoring_recommendations import _symbol_spans
+
+# Above the largest per-statement parameter cap SQLite can be built with.
+PATHS = [f"src/module_{i}/file_{i}.py" for i in range(33_000)]
+
+
+ "read",
+ [
+ lambda session: _symbol_spans(session, "repo-1", PATHS),
+ lambda session: load_tests_in_files(session, "repo-1", PATHS),
+ lambda session: covering_files(session, "repo-1", set(PATHS)),
+ ],
+ ids=["symbol_spans", "tests_in_files", "tests_covering_files"],
+)
+async def test_a_target_set_larger_than_the_parameter_cap_is_read(async_session, read):
+ assert await read(async_session) == {}
+
+
+async def test_an_update_over_more_files_than_the_parameter_cap_persists(async_session):
+ """The health persist, the plan finalize and the read snapshots of #4401.
+
+ The cap is lowered to 600 on the test connection, so 700 files go past it
+ the way a 9k-file repository goes past SQLite's 32766.
+ """
+ import sqlite3
+ from types import SimpleNamespace
+
+ from sqlalchemy import func, select
+
+ from repowise.core.analysis.health.refactoring.models import RefactoringSuggestion
+ from repowise.core.persistence.crud.analysis.actions import write_read_snapshots
+ from repowise.core.persistence.models import (
+ HealthFileMetric,
+ HealthFinding,
+ )
+ from repowise.core.persistence.models import (
+ RefactoringSuggestion as SuggestionRow,
+ )
+ from repowise.core.pipeline.incremental import persist_partial_health
+ from tests.unit.persistence.helpers import insert_repo
+
+ raw = await (await async_session.connection()).get_raw_connection()
+ raw.driver_connection._conn.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 600)
+ repo = await insert_repo(async_session)
+ paths = [f"src/file_{i}.py" for i in range(700)]
+ report = SimpleNamespace(
+ authoritative_paths=set(paths),
+ performance_authoritative_paths=set(),
+ performance_plan_policy=None,
+ metrics=[{"file_path": p, "score": 6.0, "nloc": 120} for p in paths],
+ findings=[
+ {
+ "file_path": p,
+ "biomarker_type": "low_cohesion",
+ "severity": "medium",
+ "function_name": "Foo",
+ "line_start": 1,
+ "line_end": 80,
+ "health_impact": 1.5,
+ "reason": "low cohesion",
+ }
+ for p in paths
+ ],
+ refactoring_suggestions=[
+ RefactoringSuggestion(
+ refactoring_type="extract_class",
+ file_path=p,
+ target_symbol="Foo",
+ line_start=1,
+ line_end=80,
+ plan={"groups": [{"name": "Store", "methods": ["m1", "m2"], "fields": ["a"]}]},
+ evidence={"lcom4": 2, "method_count": 6, "field_count": 2, "wmc": 9},
+ impact_delta=2.4,
+ effort_bucket="M",
+ blast_radius={"dependents_count": 3},
+ confidence="high",
+ source_biomarker="low_cohesion",
+ )
+ for p in paths
+ ],
+ function_blame_rows=[],
+ )
+
+ await persist_partial_health(async_session, repo.id, report)
+ # Twice: the second pass replaces and reconciles stored rows over every path.
+ await persist_partial_health(async_session, repo.id, report)
+ assert await write_read_snapshots(async_session, repo.id) == []
+
+ for model in (HealthFileMetric, HealthFinding, SuggestionRow):
+ count = await async_session.scalar(
+ select(func.count()).select_from(model).where(model.repository_id == repo.id)
+ )
+ assert count == len(paths), model.__name__