Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(server): honor exclude_patterns in server jobs; prune stale rows (#354)

joptimusdafde0490922
diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.py
index 454dac6b8..fc71d80d3 100644
--- a/packages/core/src/repowise/core/pipeline/persist.py
+++ b/packages/core/src/repowise/core/pipeline/persist.py
@@ -108,6 +108,114 @@ async def persist_graph_nodes(
logger.warning("graph_metrics_materialize_skipped", error=str(exc))
+# Chunk size for IN (...) deletes — stays under SQLite's host-parameter limit.
+_PRUNE_CHUNK = 500
+
+
+async def _prune_stale_file_rows(
+ session: Any,
+ repo_id: str,
+ current_graph_file_paths: set[str],
+ current_git_file_paths: set[str],
+) -> None:
+ """Delete file-scoped rows for files absent from the latest full pipeline run.
+
+ The parser and git indexer disagree on the file set — a file can be
+ git-tracked yet absent from ``parsed_files`` (parse failure, unparsed
+ extension, skipped) — so the tables use different sources of truth.
+ *current_graph_file_paths* (from ``parsed_files``) governs graph/analysis
+ tables; *current_git_file_paths* (from ``git_metadata_list``) governs
+ ``git_metadata`` only. Each set independently no-ops when empty to avoid
+ wiping rows on a broken run. FULL persistence only — not incremental paths.
+ """
+ from sqlalchemy import delete, or_, select
+
+ from repowise.core.persistence.models import (
+ DeadCodeFinding,
+ GitMetadata,
+ GraphEdge,
+ GraphMetric,
+ GraphNode,
+ HealthFileMetric,
+ HealthFinding,
+ SecurityFinding,
+ WikiSymbol,
+ )
+
+ async def _delete_stale_by_paths(model: Any, column: Any, current: set[str]) -> None:
+ # Diff persisted paths against *current* in Python so the IN (...) is
+ # bounded by the stale set, not the whole repo (SQLite param limit).
+ if not current:
+ return
+ existing = set(
+ (await session.execute(select(column).where(model.repository_id == repo_id).distinct()))
+ .scalars()
+ .all()
+ )
+ stale = [p for p in existing if p not in current]
+ for i in range(0, len(stale), _PRUNE_CHUNK):
+ await session.execute(
+ delete(model).where(
+ model.repository_id == repo_id,
+ column.in_(stale[i : i + _PRUNE_CHUNK]),
+ )
+ )
+
+ # ---- Graph nodes + edges -------------------------------------------------
+ # File nodes key on node_id; symbol nodes on file_path. Delete edges before
+ # nodes (no FK cascade between the tables).
+ if current_graph_file_paths:
+ node_rows = (
+ await session.execute(
+ select(GraphNode.node_id, GraphNode.node_type, GraphNode.file_path).where(
+ GraphNode.repository_id == repo_id
+ )
+ )
+ ).all()
+ stale_node_ids = [
+ node_id
+ for node_id, node_type, file_path in node_rows
+ if (node_type == "file" and node_id not in current_graph_file_paths)
+ or (
+ node_type != "file"
+ and file_path is not None
+ and file_path not in current_graph_file_paths
+ )
+ ]
+ for i in range(0, len(stale_node_ids), _PRUNE_CHUNK):
+ batch = stale_node_ids[i : i + _PRUNE_CHUNK]
+ await session.execute(
+ delete(GraphEdge).where(
+ GraphEdge.repository_id == repo_id,
+ or_(
+ GraphEdge.source_node_id.in_(batch),
+ GraphEdge.target_node_id.in_(batch),
+ ),
+ )
+ )
+ await session.execute(
+ delete(GraphNode).where(
+ GraphNode.repository_id == repo_id,
+ GraphNode.node_id.in_(batch),
+ )
+ )
+
+ # GraphMetric is file-level only (node_id == path).
+ await _delete_stale_by_paths(GraphMetric, GraphMetric.node_id, current_graph_file_paths)
+ await _delete_stale_by_paths(WikiSymbol, WikiSymbol.file_path, current_graph_file_paths)
+ await _delete_stale_by_paths(
+ SecurityFinding, SecurityFinding.file_path, current_graph_file_paths
+ )
+ await _delete_stale_by_paths(
+ DeadCodeFinding, DeadCodeFinding.file_path, current_graph_file_paths
+ )
+ await _delete_stale_by_paths(
+ HealthFileMetric, HealthFileMetric.file_path, current_graph_file_paths
+ )
+ await _delete_stale_by_paths(HealthFinding, HealthFinding.file_path, current_graph_file_paths)
+ await _delete_stale_by_paths(GitMetadata, GitMetadata.file_path, current_git_file_paths)
+
+
async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:
"""Persist ingestion-phase outputs: graph nodes/edges, external systems,
symbols, and the per-file security scan.
@@ -412,6 +520,18 @@ async def persist_pipeline_result(
This function mutates ``sym.file_path`` on parsed-file symbols that
lack one. Callers should treat *result* as consumed after this call.
"""
+ # Prune rows for files absent from this full result before the phase
+ # persisters re-upsert. graph/analysis tables key off parsed_files;
+ # git_metadata keys off the git indexer's set (a file can be git-tracked
+ # but unparsed). Runs only here, never in the reusable phase persisters.
+ current_graph_file_paths = {pf.file_info.path for pf in result.parsed_files}
+ current_git_file_paths = {
+ (gm if isinstance(gm, dict) else dataclasses.asdict(gm)).get("file_path", "")
+ for gm in result.git_metadata_list
+ }
+ current_git_file_paths.discard("")
+ await _prune_stale_file_rows(session, repo_id, current_graph_file_paths, current_git_file_paths)
+
symbol_count = await persist_ingestion(result, session, repo_id)
await persist_git(result, session, repo_id)
await persist_analysis(result, session, repo_id)
diff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.py
index 9add0e26a..53c58afb8 100644
--- a/packages/server/src/repowise/server/job_executor.py
+++ b/packages/server/src/repowise/server/job_executor.py
@@ -26,6 +26,47 @@ from repowise.core.pipeline import persist_pipeline_result, run_pipeline
logger = structlog.get_logger(__name__)
+
+def _repo_exclude_patterns(repo: Any, repo_path: str) -> list[str]:
+ """Collect a server job's exclude patterns from both config sources.
+
+ Web-managed repos store settings in ``Repository.settings_json``; CLI and
+ ``repowise init`` workflows write them to ``.repowise/config.yaml``. Server
+ jobs should honor either, so we merge both — order-preserving and
+ de-duplicated, settings first. A missing or malformed source is ignored
+ rather than fatal.
+ """
+ patterns: list[str] = []
+
+ def _add(values: Any) -> None:
+ if not isinstance(values, list):
+ return
+ for value in values:
+ if isinstance(value, str) and value not in patterns:
+ patterns.append(value)
+
+ # Source 1: DB-stored repo settings (web UI).
+ try:
+ settings = json.loads(getattr(repo, "settings_json", "") or "{}")
+ if isinstance(settings, dict):
+ _add(settings.get("exclude_patterns"))
+ except (TypeError, ValueError):
+ logger.debug("repo_settings_json_unparsable", repo_path=repo_path)
+
+ # Source 2: repo-local .repowise/config.yaml (CLI/init). Reuse the shared
+ # loader so we inherit its YAML + flat-format fallback handling.
+ try:
+ from repowise.core.repo_config import load_repo_config
+
+ cfg = load_repo_config(Path(repo_path))
+ if isinstance(cfg, dict):
+ _add(cfg.get("exclude_patterns"))
+ except Exception:
+ logger.debug("repo_config_yaml_unreadable", repo_path=repo_path)
+
+ return patterns
+
+
# Phase → numeric level mapping for job.current_level
_PHASE_LEVELS = {
"traverse": 0,
@@ -86,9 +127,7 @@ class JobProgressCallback:
self._sync_job_status()
def on_message(self, level: str, text: str) -> None:
- getattr(logger, level, logger.info)(
- text, job_id=self._job_id, phase=self._phase
- )
+ getattr(logger, level, logger.info)(text, job_id=self._job_id, phase=self._phase)
def _sync_job_status(self, *, force: bool = False) -> None:
"""Fire-and-forget progress update in the current event loop.
@@ -211,11 +250,16 @@ async def execute_job(
repo = await get_repository(session, job.repository_id)
if repo is None:
logger.error("repo_not_found", job_id=job_id, repo_id=job.repository_id)
- await update_job_status(session, job_id, "failed", error_message="Repository not found")
+ await update_job_status(
+ session, job_id, "failed", error_message="Repository not found"
+ )
return
repo_path = repo.local_path
repo_id = repo.id
+ # Resolve excludes while ``repo`` is still session-attached. Every
+ # job entry point flows through here, so this covers them all.
+ exclude_patterns = _repo_exclude_patterns(repo, repo_path)
config = json.loads(job.config_json) if job.config_json else {}
is_full_resync = config.get("mode") == "full_resync"
@@ -248,6 +292,7 @@ async def execute_job(
llm_client=llm_client,
vector_store=vector_store,
progress=progress,
+ exclude_patterns=exclude_patterns or None,
)
# ---- Incremental page regeneration for sync mode ------------------
@@ -257,7 +302,11 @@ async def execute_job(
incremental_pages: list = []
if not is_full_resync and llm_client is not None:
incremental_pages = await _incremental_page_regen(
- Path(repo_path), result, llm_client, config, progress,
+ Path(repo_path),
+ result,
+ llm_client,
+ config,
+ progress,
)
# ---- Persist results -----------------------------------------------
@@ -439,7 +488,9 @@ async def _incremental_page_regen(
return []
cascade_budget = compute_adaptive_budget(file_diffs, result.file_count)
- affected = detector.get_affected_pages(file_diffs, result.graph_builder.graph(), cascade_budget)
+ affected = detector.get_affected_pages(
+ file_diffs, result.graph_builder.graph(), cascade_budget
+ )
if not affected.regenerate:
logger.info("incremental_page_regen_skipped", reason="no_affected_pages")
diff --git a/tests/unit/persistence/test_persist_prune.py b/tests/unit/persistence/test_persist_prune.py
new file mode 100644
index 000000000..0161f6c5f
--- /dev/null
+++ b/tests/unit/persistence/test_persist_prune.py
@@ -0,0 +1,194 @@
+"""Tests for full-run stale file-scoped row pruning.
+
+A full pipeline result is authoritative for which files exist. When a file
+disappears from the latest result (e.g. a directory newly added to
+``exclude_patterns``), its previously-persisted rows must be pruned so stale
+graph / dead-code / health / git data does not leak into those surfaces.
+"""
+
+from __future__ import annotations
+
+from sqlalchemy import select
+
+from repowise.core.persistence.models import (
+ DeadCodeFinding,
+ GitMetadata,
+ GraphEdge,
+ GraphMetric,
+ GraphNode,
+ HealthFileMetric,
+ HealthFinding,
+ SecurityFinding,
+ WikiSymbol,
+)
+from repowise.core.pipeline.persist import _prune_stale_file_rows
+from tests.unit.persistence.helpers import insert_repo
+
+STALE = "tools/debug.js"
+KEPT = "src/app.js"
+
+
+async def _seed(session, repo_id: str) -> None:
+ """Seed file-scoped rows for both a stale and a kept file."""
+ for path in (STALE, KEPT):
+ # graph: one file node + one symbol node owned by the file
+ session.add(GraphNode(repository_id=repo_id, node_id=path, node_type="file"))
+ session.add(
+ GraphNode(
+ repository_id=repo_id,
+ node_id=f"{path}::main",
+ node_type="symbol",
+ file_path=path,
+ )
+ )
+ session.add(GraphMetric(repository_id=repo_id, node_id=path))
+ session.add(
+ WikiSymbol(
+ repository_id=repo_id,
+ file_path=path,
+ symbol_id=f"{path}::main",
+ name="main",
+ qualified_name="main",
+ kind="function",
+ )
+ )
+ session.add(
+ SecurityFinding(
+ repository_id=repo_id,
+ file_path=path,
+ kind="hardcoded_secret",
+ severity="high",
+ )
+ )
+ session.add(DeadCodeFinding(repository_id=repo_id, kind="unreachable_file", file_path=path))
+ session.add(HealthFileMetric(repository_id=repo_id, file_path=path))
+ session.add(
+ HealthFinding(
+ repository_id=repo_id,
+ file_path=path,
+ biomarker_type="complexity",
+ severity="low",
+ )
+ )
+ session.add(GitMetadata(repository_id=repo_id, file_path=path))
+
+ # An edge between the two files (references the stale file node).
+ session.add(
+ GraphEdge(
+ repository_id=repo_id,
+ source_node_id=KEPT,
+ target_node_id=STALE,
+ edge_type="imports",
+ )
+ )
+ # An edge that survives (both endpoints kept).
+ session.add(
+ GraphEdge(
+ repository_id=repo_id,
+ source_node_id=KEPT,
+ target_node_id=f"{KEPT}::main",
+ edge_type="contains",
+ )
+ )
+ await session.flush()
+
+
+async def _paths(session, model, column, repo_id) -> set[str]:
+ rows = (
+ (await session.execute(select(column).where(model.repository_id == repo_id)))
+ .scalars()
+ .all()
+ )
+ return set(rows)
+
+
+async def test_prune_removes_stale_file_rows(async_session):
+ repo = await insert_repo(async_session)
+ await _seed(async_session, repo.id)
+
+ await _prune_stale_file_rows(async_session, repo.id, {KEPT}, {KEPT})
+ await async_session.commit()
+
+ # graph nodes: stale file node + its symbol node gone; kept ones remain.
+ node_ids = await _paths(async_session, GraphNode, GraphNode.node_id, repo.id)
+ assert node_ids == {KEPT, f"{KEPT}::main"}
+
+ # graph edges referencing the stale node are gone; the kept edge survives.
+ edges = (
+ (await async_session.execute(select(GraphEdge).where(GraphEdge.repository_id == repo.id)))
+ .scalars()
+ .all()
+ )
+ assert [(e.source_node_id, e.target_node_id) for e in edges] == [(KEPT, f"{KEPT}::main")]
+
+ # Every remaining file-scoped table holds only the kept path.
+ assert await _paths(async_session, GraphMetric, GraphMetric.node_id, repo.id) == {KEPT}
+ assert await _paths(async_session, WikiSymbol, WikiSymbol.file_path, repo.id) == {KEPT}
+ assert await _paths(async_session, SecurityFinding, SecurityFinding.file_path, repo.id) == {
+ KEPT
+ }
+ assert await _paths(async_session, DeadCodeFinding, DeadCodeFinding.file_path, repo.id) == {
+ KEPT
+ }
+ assert await _paths(async_session, HealthFileMetric, HealthFileMetric.file_path, repo.id) == {
+ KEPT
+ }
+ assert await _paths(async_session, HealthFinding, HealthFinding.file_path, repo.id) == {KEPT}
+ assert await _paths(async_session, GitMetadata, GitMetadata.file_path, repo.id) == {KEPT}
+
+
+async def test_prune_noop_on_empty_current_paths(async_session):
+ """Empty current set must NOT wipe the repo (guards broken pipeline runs)."""
+ repo = await insert_repo(async_session)
+ await _seed(async_session, repo.id)
+
+ await _prune_stale_file_rows(async_session, repo.id, set(), set())
+ await async_session.commit()
+
+ # All rows for both files still present.
+ assert await _paths(async_session, GitMetadata, GitMetadata.file_path, repo.id) == {
+ STALE,
+ KEPT,
+ }
+ node_ids = await _paths(async_session, GraphNode, GraphNode.node_id, repo.id)
+ assert STALE in node_ids and KEPT in node_ids
+
+
+async def test_prune_scoped_to_repo(async_session):
+ """Pruning one repo must not touch another repo's rows."""
+ repo_a = await insert_repo(async_session, local_path="/tmp/repo-a", name="a")
+ repo_b = await insert_repo(async_session, local_path="/tmp/repo-b", name="b")
+ await _seed(async_session, repo_a.id)
+ await _seed(async_session, repo_b.id)
+
+ await _prune_stale_file_rows(async_session, repo_a.id, {KEPT}, {KEPT})
+ await async_session.commit()
+
+ # repo_b is untouched.
+ assert await _paths(async_session, GitMetadata, GitMetadata.file_path, repo_b.id) == {
+ STALE,
+ KEPT,
+ }
+
+
+async def test_prune_keeps_git_indexed_unparsed_file(async_session):
+ """git_metadata keys off the git set, so a git-tracked unparsed file survives."""
+ unparsed = "src/unparsed-but-git-indexed.js"
+ repo = await insert_repo(async_session)
+ await _seed(async_session, repo.id)
+ async_session.add(GitMetadata(repository_id=repo.id, file_path=unparsed))
+ await async_session.flush()
+
+ # Graph set lacks the unparsed file; git set includes it.
+ await _prune_stale_file_rows(async_session, repo.id, {KEPT}, {KEPT, unparsed})
+ await async_session.commit()
+
+ assert await _paths(async_session, GitMetadata, GitMetadata.file_path, repo.id) == {
+ KEPT,
+ unparsed,
+ }
+ # It was never parsed, so it has no graph node either way.
+ assert await _paths(async_session, GraphNode, GraphNode.node_id, repo.id) == {
+ KEPT,
+ f"{KEPT}::main",
+ }
diff --git a/tests/unit/server/test_job_executor.py b/tests/unit/server/test_job_executor.py
new file mode 100644
index 000000000..6e17479aa
--- /dev/null
+++ b/tests/unit/server/test_job_executor.py
@@ -0,0 +1,156 @@
+"""Tests for the background job executor's exclude-pattern handling.
+
+Server-triggered jobs (web sync, full-resync, workspace sync, webhooks,
+scheduler) all route through ``execute_job``. These tests prove that the
+repository's ``exclude_patterns`` reach ``run_pipeline`` so excluded paths
+are not re-indexed.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from repowise.core.persistence.crud import upsert_generation_job, upsert_repository
+from repowise.server.job_executor import _repo_exclude_patterns, execute_job
+
+
+def _fake_result() -> SimpleNamespace:
+ """Minimal stand-in for a PipelineResult (persist is mocked out)."""
+ return SimpleNamespace(
+ generated_pages=[],
+ parsed_files=[],
+ file_count=1,
+ symbol_count=2,
+ )
+
+
+async def _seed_repo_and_job(
+ session_factory,
+ repo_path,
+ *,
+ settings: dict | None = None,
+) -> str:
+ """Insert a repo (with settings) + a pending full_resync job; return job_id."""
+ async with session_factory() as session:
+ repo = await upsert_repository(
+ session,
+ name="test-repo",
+ local_path=str(repo_path),
+ settings=settings or {},
+ )
+ job = await upsert_generation_job(
+ session,
+ repository_id=repo.id,
+ config={"mode": "full_resync"},
+ )
+ await session.commit()
+ return job.id
+
+
+async def test_execute_job_passes_exclude_patterns_from_settings(session_factory, tmp_path):
+ """settings_json exclude_patterns must reach run_pipeline."""
+ job_id = await _seed_repo_and_job(
+ session_factory,
+ tmp_path,
+ settings={"exclude_patterns": ["tools/", "node_modules/"]},
+ )
+
+ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)
+
+ run_pipeline_mock = AsyncMock(return_value=_fake_result())
+ with (
+ patch("repowise.server.job_executor.run_pipeline", run_pipeline_mock),
+ patch("repowise.server.job_executor.persist_pipeline_result", AsyncMock()),
+ patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ side_effect=RuntimeError("no provider"),
+ ),
+ ):
+ await execute_job(job_id, app_state)
+
+ run_pipeline_mock.assert_awaited_once()
+ assert run_pipeline_mock.await_args.kwargs["exclude_patterns"] == [
+ "tools/",
+ "node_modules/",
+ ]
+
+
+async def test_execute_job_no_excludes_passes_none(session_factory, tmp_path):
+ """With no configured excludes, run_pipeline receives None (not [])."""
+ job_id = await _seed_repo_and_job(session_factory, tmp_path, settings={})
+
+ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)
+
+ run_pipeline_mock = AsyncMock(return_value=_fake_result())
+ with (
+ patch("repowise.server.job_executor.run_pipeline", run_pipeline_mock),
+ patch("repowise.server.job_executor.persist_pipeline_result", AsyncMock()),
+ patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ side_effect=RuntimeError("no provider"),
+ ),
+ ):
+ await execute_job(job_id, app_state)
+
+ run_pipeline_mock.assert_awaited_once()
+ assert run_pipeline_mock.await_args.kwargs["exclude_patterns"] is None
+
+
+def test_repo_exclude_patterns_merges_settings_and_config(tmp_path):
+ """DB settings + .repowise/config.yaml merge, order-preserved & de-duped."""
+ import json
+
+ config_dir = tmp_path / ".repowise"
+ config_dir.mkdir()
+ (config_dir / "config.yaml").write_text(
+ "exclude_patterns:\n - node_modules/\n - vendor/\n", encoding="utf-8"
+ )
+
+ repo = SimpleNamespace(
+ settings_json=json.dumps({"exclude_patterns": ["tools/", "node_modules/"]})
+ )
+
+ patterns = _repo_exclude_patterns(repo, str(tmp_path))
+
+ assert patterns == ["tools/", "node_modules/", "vendor/"]
+
+
+def test_repo_exclude_patterns_ignores_malformed_sources(tmp_path):
+ """Malformed settings_json / missing config are ignored, not fatal."""
+ repo = SimpleNamespace(settings_json="{not valid json")
+ assert _repo_exclude_patterns(repo, str(tmp_path)) == []
+
+
+async def test_execute_job_merges_config_yaml_excludes(session_factory, tmp_path):
+ """End-to-end regression: the real user case (tools/ excluded).
+
+ Repo settings carry ``exclude_patterns: ["tools/"]``; the job path must
+ forward exactly that to run_pipeline so ``tools/`` is never re-indexed.
+ """
+ job_id = await _seed_repo_and_job(
+ session_factory,
+ tmp_path,
+ settings={"exclude_patterns": ["tools/"]},
+ )
+
+ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)
+
+ run_pipeline_mock = AsyncMock(return_value=_fake_result())
+ with (
+ patch("repowise.server.job_executor.run_pipeline", run_pipeline_mock),
+ patch("repowise.server.job_executor.persist_pipeline_result", AsyncMock()),
+ patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ side_effect=RuntimeError("no provider"),
+ ),
+ ):
+ await execute_job(job_id, app_state)
+
+ run_pipeline_mock.assert_awaited_once()
+ assert run_pipeline_mock.await_args.kwargs["exclude_patterns"] == ["tools/"]