Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(server): honor exclude_patterns in server jobs; prune stale rows (#354)
joptimusdafde0490922
diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex 454dac6b8..fc71d80d3 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -108,6 +108,114 @@ async def persist_graph_nodes(logger.warning("graph_metrics_materialize_skipped", error=str(exc))+# Chunk size for IN (...) deletes — stays under SQLite's host-parameter limit.+_PRUNE_CHUNK = 500+++async def _prune_stale_file_rows(+ session: Any,+ repo_id: str,+ current_graph_file_paths: set[str],+ current_git_file_paths: set[str],+) -> None:+ """Delete file-scoped rows for files absent from the latest full pipeline run.++ The parser and git indexer disagree on the file set — a file can be+ git-tracked yet absent from ``parsed_files`` (parse failure, unparsed+ extension, skipped) — so the tables use different sources of truth.+ *current_graph_file_paths* (from ``parsed_files``) governs graph/analysis+ tables; *current_git_file_paths* (from ``git_metadata_list``) governs+ ``git_metadata`` only. Each set independently no-ops when empty to avoid+ wiping rows on a broken run. FULL persistence only — not incremental paths.+ """+ from sqlalchemy import delete, or_, select++ from repowise.core.persistence.models import (+ DeadCodeFinding,+ GitMetadata,+ GraphEdge,+ GraphMetric,+ GraphNode,+ HealthFileMetric,+ HealthFinding,+ SecurityFinding,+ WikiSymbol,+ )++ async def _delete_stale_by_paths(model: Any, column: Any, current: set[str]) -> None:+ # Diff persisted paths against *current* in Python so the IN (...) is+ # bounded by the stale set, not the whole repo (SQLite param limit).+ if not current:+ return+ existing = set(+ (await session.execute(select(column).where(model.repository_id == repo_id).distinct()))+ .scalars()+ .all()+ )+ stale = [p for p in existing if p not in current]+ for i in range(0, len(stale), _PRUNE_CHUNK):+ await session.execute(+ delete(model).where(+ model.repository_id == repo_id,+ column.in_(stale[i : i + _PRUNE_CHUNK]),+ )+ )++ # ---- Graph nodes + edges -------------------------------------------------+ # File nodes key on node_id; symbol nodes on file_path. Delete edges before+ # nodes (no FK cascade between the tables).+ if current_graph_file_paths:+ node_rows = (+ await session.execute(+ select(GraphNode.node_id, GraphNode.node_type, GraphNode.file_path).where(+ GraphNode.repository_id == repo_id+ )+ )+ ).all()+ stale_node_ids = [+ node_id+ for node_id, node_type, file_path in node_rows+ if (node_type == "file" and node_id not in current_graph_file_paths)+ or (+ node_type != "file"+ and file_path is not None+ and file_path not in current_graph_file_paths+ )+ ]+ for i in range(0, len(stale_node_ids), _PRUNE_CHUNK):+ batch = stale_node_ids[i : i + _PRUNE_CHUNK]+ await session.execute(+ delete(GraphEdge).where(+ GraphEdge.repository_id == repo_id,+ or_(+ GraphEdge.source_node_id.in_(batch),+ GraphEdge.target_node_id.in_(batch),+ ),+ )+ )+ await session.execute(+ delete(GraphNode).where(+ GraphNode.repository_id == repo_id,+ GraphNode.node_id.in_(batch),+ )+ )++ # GraphMetric is file-level only (node_id == path).+ await _delete_stale_by_paths(GraphMetric, GraphMetric.node_id, current_graph_file_paths)+ await _delete_stale_by_paths(WikiSymbol, WikiSymbol.file_path, current_graph_file_paths)+ await _delete_stale_by_paths(+ SecurityFinding, SecurityFinding.file_path, current_graph_file_paths+ )+ await _delete_stale_by_paths(+ DeadCodeFinding, DeadCodeFinding.file_path, current_graph_file_paths+ )+ await _delete_stale_by_paths(+ HealthFileMetric, HealthFileMetric.file_path, current_graph_file_paths+ )+ await _delete_stale_by_paths(HealthFinding, HealthFinding.file_path, current_graph_file_paths)+ await _delete_stale_by_paths(GitMetadata, GitMetadata.file_path, current_git_file_paths)++async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:"""Persist ingestion-phase outputs: graph nodes/edges, external systems,symbols, and the per-file security scan.@@ -412,6 +520,18 @@ async def persist_pipeline_result(This function mutates ``sym.file_path`` on parsed-file symbols thatlack one. Callers should treat *result* as consumed after this call."""+ # Prune rows for files absent from this full result before the phase+ # persisters re-upsert. graph/analysis tables key off parsed_files;+ # git_metadata keys off the git indexer's set (a file can be git-tracked+ # but unparsed). Runs only here, never in the reusable phase persisters.+ current_graph_file_paths = {pf.file_info.path for pf in result.parsed_files}+ current_git_file_paths = {+ (gm if isinstance(gm, dict) else dataclasses.asdict(gm)).get("file_path", "")+ for gm in result.git_metadata_list+ }+ current_git_file_paths.discard("")+ await _prune_stale_file_rows(session, repo_id, current_graph_file_paths, current_git_file_paths)+symbol_count = await persist_ingestion(result, session, repo_id)await persist_git(result, session, repo_id)await persist_analysis(result, session, repo_id)diff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.pyindex 9add0e26a..53c58afb8 100644--- a/packages/server/src/repowise/server/job_executor.py+++ b/packages/server/src/repowise/server/job_executor.py@@ -26,6 +26,47 @@ from repowise.core.pipeline import persist_pipeline_result, run_pipelinelogger = structlog.get_logger(__name__)++def _repo_exclude_patterns(repo: Any, repo_path: str) -> list[str]:+ """Collect a server job's exclude patterns from both config sources.++ Web-managed repos store settings in ``Repository.settings_json``; CLI and+ ``repowise init`` workflows write them to ``.repowise/config.yaml``. Server+ jobs should honor either, so we merge both — order-preserving and+ de-duplicated, settings first. A missing or malformed source is ignored+ rather than fatal.+ """+ patterns: list[str] = []++ def _add(values: Any) -> None:+ if not isinstance(values, list):+ return+ for value in values:+ if isinstance(value, str) and value not in patterns:+ patterns.append(value)++ # Source 1: DB-stored repo settings (web UI).+ try:+ settings = json.loads(getattr(repo, "settings_json", "") or "{}")+ if isinstance(settings, dict):+ _add(settings.get("exclude_patterns"))+ except (TypeError, ValueError):+ logger.debug("repo_settings_json_unparsable", repo_path=repo_path)++ # Source 2: repo-local .repowise/config.yaml (CLI/init). Reuse the shared+ # loader so we inherit its YAML + flat-format fallback handling.+ try:+ from repowise.core.repo_config import load_repo_config++ cfg = load_repo_config(Path(repo_path))+ if isinstance(cfg, dict):+ _add(cfg.get("exclude_patterns"))+ except Exception:+ logger.debug("repo_config_yaml_unreadable", repo_path=repo_path)++ return patterns++# Phase → numeric level mapping for job.current_level_PHASE_LEVELS = {"traverse": 0,@@ -86,9 +127,7 @@ class JobProgressCallback:self._sync_job_status()def on_message(self, level: str, text: str) -> None:- getattr(logger, level, logger.info)(- text, job_id=self._job_id, phase=self._phase- )+ getattr(logger, level, logger.info)(text, job_id=self._job_id, phase=self._phase)def _sync_job_status(self, *, force: bool = False) -> None:"""Fire-and-forget progress update in the current event loop.@@ -211,11 +250,16 @@ async def execute_job(repo = await get_repository(session, job.repository_id)if repo is None:logger.error("repo_not_found", job_id=job_id, repo_id=job.repository_id)- await update_job_status(session, job_id, "failed", error_message="Repository not found")+ await update_job_status(+ session, job_id, "failed", error_message="Repository not found"+ )returnrepo_path = repo.local_pathrepo_id = repo.id+ # Resolve excludes while ``repo`` is still session-attached. Every+ # job entry point flows through here, so this covers them all.+ exclude_patterns = _repo_exclude_patterns(repo, repo_path)config = json.loads(job.config_json) if job.config_json else {}is_full_resync = config.get("mode") == "full_resync"@@ -248,6 +292,7 @@ async def execute_job(llm_client=llm_client,vector_store=vector_store,progress=progress,+ exclude_patterns=exclude_patterns or None,)# ---- Incremental page regeneration for sync mode ------------------@@ -257,7 +302,11 @@ async def execute_job(incremental_pages: list = []if not is_full_resync and llm_client is not None:incremental_pages = await _incremental_page_regen(- Path(repo_path), result, llm_client, config, progress,+ Path(repo_path),+ result,+ llm_client,+ config,+ progress,)# ---- Persist results -----------------------------------------------@@ -439,7 +488,9 @@ async def _incremental_page_regen(return []cascade_budget = compute_adaptive_budget(file_diffs, result.file_count)- affected = detector.get_affected_pages(file_diffs, result.graph_builder.graph(), cascade_budget)+ affected = detector.get_affected_pages(+ file_diffs, result.graph_builder.graph(), cascade_budget+ )if not affected.regenerate:logger.info("incremental_page_regen_skipped", reason="no_affected_pages")diff --git a/tests/unit/persistence/test_persist_prune.py b/tests/unit/persistence/test_persist_prune.pynew file mode 100644index 000000000..0161f6c5f--- /dev/null+++ b/tests/unit/persistence/test_persist_prune.py@@ -0,0 +1,194 @@+"""Tests for full-run stale file-scoped row pruning.++A full pipeline result is authoritative for which files exist. When a file+disappears from the latest result (e.g. a directory newly added to+``exclude_patterns``), its previously-persisted rows must be pruned so stale+graph / dead-code / health / git data does not leak into those surfaces.+"""++from __future__ import annotations++from sqlalchemy import select++from repowise.core.persistence.models import (+ DeadCodeFinding,+ GitMetadata,+ GraphEdge,+ GraphMetric,+ GraphNode,+ HealthFileMetric,+ HealthFinding,+ SecurityFinding,+ WikiSymbol,+)+from repowise.core.pipeline.persist import _prune_stale_file_rows+from tests.unit.persistence.helpers import insert_repo++STALE = "tools/debug.js"+KEPT = "src/app.js"+++async def _seed(session, repo_id: str) -> None:+ """Seed file-scoped rows for both a stale and a kept file."""+ for path in (STALE, KEPT):+ # graph: one file node + one symbol node owned by the file+ session.add(GraphNode(repository_id=repo_id, node_id=path, node_type="file"))+ session.add(+ GraphNode(+ repository_id=repo_id,+ node_id=f"{path}::main",+ node_type="symbol",+ file_path=path,+ )+ )+ session.add(GraphMetric(repository_id=repo_id, node_id=path))+ session.add(+ WikiSymbol(+ repository_id=repo_id,+ file_path=path,+ symbol_id=f"{path}::main",+ name="main",+ qualified_name="main",+ kind="function",+ )+ )+ session.add(+ SecurityFinding(+ repository_id=repo_id,+ file_path=path,+ kind="hardcoded_secret",+ severity="high",+ )+ )+ session.add(DeadCodeFinding(repository_id=repo_id, kind="unreachable_file", file_path=path))+ session.add(HealthFileMetric(repository_id=repo_id, file_path=path))+ session.add(+ HealthFinding(+ repository_id=repo_id,+ file_path=path,+ biomarker_type="complexity",+ severity="low",+ )+ )+ session.add(GitMetadata(repository_id=repo_id, file_path=path))++ # An edge between the two files (references the stale file node).+ session.add(+ GraphEdge(+ repository_id=repo_id,+ source_node_id=KEPT,+ target_node_id=STALE,+ edge_type="imports",+ )+ )+ # An edge that survives (both endpoints kept).+ session.add(+ GraphEdge(+ repository_id=repo_id,+ source_node_id=KEPT,+ target_node_id=f"{KEPT}::main",+ edge_type="contains",+ )+ )+ await session.flush()+++async def _paths(session, model, column, repo_id) -> set[str]:+ rows = (+ (await session.execute(select(column).where(model.repository_id == repo_id)))+ .scalars()+ .all()+ )+ return set(rows)+++async def test_prune_removes_stale_file_rows(async_session):+ repo = await insert_repo(async_session)+ await _seed(async_session, repo.id)++ await _prune_stale_file_rows(async_session, repo.id, {KEPT}, {KEPT})+ await async_session.commit()++ # graph nodes: stale file node + its symbol node gone; kept ones remain.+ node_ids = await _paths(async_session, GraphNode, GraphNode.node_id, repo.id)+ assert node_ids == {KEPT, f"{KEPT}::main"}++ # graph edges referencing the stale node are gone; the kept edge survives.+ edges = (+ (await async_session.execute(select(GraphEdge).where(GraphEdge.repository_id == repo.id)))+ .scalars()+ .all()+ )+ assert [(e.source_node_id, e.target_node_id) for e in edges] == [(KEPT, f"{KEPT}::main")]++ # Every remaining file-scoped table holds only the kept path.+ assert await _paths(async_session, GraphMetric, GraphMetric.node_id, repo.id) == {KEPT}+ assert await _paths(async_session, WikiSymbol, WikiSymbol.file_path, repo.id) == {KEPT}+ assert await _paths(async_session, SecurityFinding, SecurityFinding.file_path, repo.id) == {+ KEPT+ }+ assert await _paths(async_session, DeadCodeFinding, DeadCodeFinding.file_path, repo.id) == {+ KEPT+ }+ assert await _paths(async_session, HealthFileMetric, HealthFileMetric.file_path, repo.id) == {+ KEPT+ }+ assert await _paths(async_session, HealthFinding, HealthFinding.file_path, repo.id) == {KEPT}+ assert await _paths(async_session, GitMetadata, GitMetadata.file_path, repo.id) == {KEPT}+++async def test_prune_noop_on_empty_current_paths(async_session):+ """Empty current set must NOT wipe the repo (guards broken pipeline runs)."""+ repo = await insert_repo(async_session)+ await _seed(async_session, repo.id)++ await _prune_stale_file_rows(async_session, repo.id, set(), set())+ await async_session.commit()++ # All rows for both files still present.+ assert await _paths(async_session, GitMetadata, GitMetadata.file_path, repo.id) == {+ STALE,+ KEPT,+ }+ node_ids = await _paths(async_session, GraphNode, GraphNode.node_id, repo.id)+ assert STALE in node_ids and KEPT in node_ids+++async def test_prune_scoped_to_repo(async_session):+ """Pruning one repo must not touch another repo's rows."""+ repo_a = await insert_repo(async_session, local_path="/tmp/repo-a", name="a")+ repo_b = await insert_repo(async_session, local_path="/tmp/repo-b", name="b")+ await _seed(async_session, repo_a.id)+ await _seed(async_session, repo_b.id)++ await _prune_stale_file_rows(async_session, repo_a.id, {KEPT}, {KEPT})+ await async_session.commit()++ # repo_b is untouched.+ assert await _paths(async_session, GitMetadata, GitMetadata.file_path, repo_b.id) == {+ STALE,+ KEPT,+ }+++async def test_prune_keeps_git_indexed_unparsed_file(async_session):+ """git_metadata keys off the git set, so a git-tracked unparsed file survives."""+ unparsed = "src/unparsed-but-git-indexed.js"+ repo = await insert_repo(async_session)+ await _seed(async_session, repo.id)+ async_session.add(GitMetadata(repository_id=repo.id, file_path=unparsed))+ await async_session.flush()++ # Graph set lacks the unparsed file; git set includes it.+ await _prune_stale_file_rows(async_session, repo.id, {KEPT}, {KEPT, unparsed})+ await async_session.commit()++ assert await _paths(async_session, GitMetadata, GitMetadata.file_path, repo.id) == {+ KEPT,+ unparsed,+ }+ # It was never parsed, so it has no graph node either way.+ assert await _paths(async_session, GraphNode, GraphNode.node_id, repo.id) == {+ KEPT,+ f"{KEPT}::main",+ }diff --git a/tests/unit/server/test_job_executor.py b/tests/unit/server/test_job_executor.pynew file mode 100644index 000000000..6e17479aa--- /dev/null+++ b/tests/unit/server/test_job_executor.py@@ -0,0 +1,156 @@+"""Tests for the background job executor's exclude-pattern handling.++Server-triggered jobs (web sync, full-resync, workspace sync, webhooks,+scheduler) all route through ``execute_job``. These tests prove that the+repository's ``exclude_patterns`` reach ``run_pipeline`` so excluded paths+are not re-indexed.+"""++from __future__ import annotations++from types import SimpleNamespace+from unittest.mock import AsyncMock, patch++import pytest++from repowise.core.persistence.crud import upsert_generation_job, upsert_repository+from repowise.server.job_executor import _repo_exclude_patterns, execute_job+++def _fake_result() -> SimpleNamespace:+ """Minimal stand-in for a PipelineResult (persist is mocked out)."""+ return SimpleNamespace(+ generated_pages=[],+ parsed_files=[],+ file_count=1,+ symbol_count=2,+ )+++async def _seed_repo_and_job(+ session_factory,+ repo_path,+ *,+ settings: dict | None = None,+) -> str:+ """Insert a repo (with settings) + a pending full_resync job; return job_id."""+ async with session_factory() as session:+ repo = await upsert_repository(+ session,+ name="test-repo",+ local_path=str(repo_path),+ settings=settings or {},+ )+ job = await upsert_generation_job(+ session,+ repository_id=repo.id,+ config={"mode": "full_resync"},+ )+ await session.commit()+ return job.id+++async def test_execute_job_passes_exclude_patterns_from_settings(session_factory, tmp_path):+ """settings_json exclude_patterns must reach run_pipeline."""+ job_id = await _seed_repo_and_job(+ session_factory,+ tmp_path,+ settings={"exclude_patterns": ["tools/", "node_modules/"]},+ )++ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)++ run_pipeline_mock = AsyncMock(return_value=_fake_result())+ with (+ patch("repowise.server.job_executor.run_pipeline", run_pipeline_mock),+ patch("repowise.server.job_executor.persist_pipeline_result", AsyncMock()),+ patch(+ "repowise.server.provider_config.get_chat_provider_instance",+ side_effect=RuntimeError("no provider"),+ ),+ ):+ await execute_job(job_id, app_state)++ run_pipeline_mock.assert_awaited_once()+ assert run_pipeline_mock.await_args.kwargs["exclude_patterns"] == [+ "tools/",+ "node_modules/",+ ]+++async def test_execute_job_no_excludes_passes_none(session_factory, tmp_path):+ """With no configured excludes, run_pipeline receives None (not [])."""+ job_id = await _seed_repo_and_job(session_factory, tmp_path, settings={})++ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)++ run_pipeline_mock = AsyncMock(return_value=_fake_result())+ with (+ patch("repowise.server.job_executor.run_pipeline", run_pipeline_mock),+ patch("repowise.server.job_executor.persist_pipeline_result", AsyncMock()),+ patch(+ "repowise.server.provider_config.get_chat_provider_instance",+ side_effect=RuntimeError("no provider"),+ ),+ ):+ await execute_job(job_id, app_state)++ run_pipeline_mock.assert_awaited_once()+ assert run_pipeline_mock.await_args.kwargs["exclude_patterns"] is None+++def test_repo_exclude_patterns_merges_settings_and_config(tmp_path):+ """DB settings + .repowise/config.yaml merge, order-preserved & de-duped."""+ import json++ config_dir = tmp_path / ".repowise"+ config_dir.mkdir()+ (config_dir / "config.yaml").write_text(+ "exclude_patterns:\n - node_modules/\n - vendor/\n", encoding="utf-8"+ )++ repo = SimpleNamespace(+ settings_json=json.dumps({"exclude_patterns": ["tools/", "node_modules/"]})+ )++ patterns = _repo_exclude_patterns(repo, str(tmp_path))++ assert patterns == ["tools/", "node_modules/", "vendor/"]+++def test_repo_exclude_patterns_ignores_malformed_sources(tmp_path):+ """Malformed settings_json / missing config are ignored, not fatal."""+ repo = SimpleNamespace(settings_json="{not valid json")+ assert _repo_exclude_patterns(repo, str(tmp_path)) == []+++async def test_execute_job_merges_config_yaml_excludes(session_factory, tmp_path):+ """End-to-end regression: the real user case (tools/ excluded).++ Repo settings carry ``exclude_patterns: ["tools/"]``; the job path must+ forward exactly that to run_pipeline so ``tools/`` is never re-indexed.+ """+ job_id = await _seed_repo_and_job(+ session_factory,+ tmp_path,+ settings={"exclude_patterns": ["tools/"]},+ )++ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)++ run_pipeline_mock = AsyncMock(return_value=_fake_result())+ with (+ patch("repowise.server.job_executor.run_pipeline", run_pipeline_mock),+ patch("repowise.server.job_executor.persist_pipeline_result", AsyncMock()),+ patch(+ "repowise.server.provider_config.get_chat_provider_instance",+ side_effect=RuntimeError("no provider"),+ ),+ ):+ await execute_job(job_id, app_state)++ run_pipeline_mock.assert_awaited_once()+ assert run_pipeline_mock.await_args.kwargs["exclude_patterns"] == ["tools/"]