Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(mcp,rag,git): search ranking + embedding pipeline, risk calibration, co-change persistence, skeleton default, stat-only distill (#414)

Raghav Chamadiya3d175657aa5f
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd.py b/packages/cli/src/repowise/cli/commands/update_cmd.py
index 9aa01b9a4..2413bbc12 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd.py
@@ -1165,10 +1165,19 @@ def update_command(
except Exception:
prior_pages = {}
- # Generate affected pages
+ # Generate affected pages. The vector store (shared with the decision
+ # pass above) must ride along: without it regenerated pages are never
+ # re-embedded, so every update silently evicted its pages from semantic
+ # search — on long-lived repos the LanceDB corpus ended up holding
+ # decisions and structural pages but zero current file pages.
assembler = ContextAssembler(config)
generator = PageGenerator(
- provider, assembler, config, language=config.language, prior_pages=prior_pages
+ provider,
+ assembler,
+ config,
+ vector_store=decision_vector_store,
+ language=config.language,
+ prior_pages=prior_pages,
)
repo_name = repo_path.name
diff --git a/packages/core/src/repowise/core/analysis/pr_blast.py b/packages/core/src/repowise/core/analysis/pr_blast.py
index 4d645bfd2..c1f45738a 100644
--- a/packages/core/src/repowise/core/analysis/pr_blast.py
+++ b/packages/core/src/repowise/core/analysis/pr_blast.py
@@ -15,6 +15,7 @@ co_change_partners_json field stored in git_metadata rows.
from __future__ import annotations
import json
+import math
import os
from collections import defaultdict
from typing import Any
@@ -281,7 +282,20 @@ class PRBlastRadiusAnalyzer:
direct_risks: list[dict],
transitive_affected: list[dict],
) -> float:
- """Compute overall risk score on 0-10 scale."""
+ """Compute overall risk score on 0-10 scale.
+
+ Per-file risk is ``pagerank * (1 + temporal_hotspot)`` — unbounded
+ and pagerank-scaled (typically 0-0.3). The old ``min(raw * 100, 10)``
+ normalisation clipped *everything*: the 0-1 breadth bonus alone
+ scaled to 0-20 points, so any PR with >=20 transitive dependents —
+ i.e. any PR touching a hotspot — reported exactly 10.0 and the score
+ carried no information.
+
+ Instead, squash the pagerank-scale file term through an exponential
+ CDF onto 0-8 points (saturating only asymptotically) and let breadth
+ add up to 2 points. Reference points for the file term:
+ combined 0.01 -> ~0.8, 0.05 -> ~3.1, 0.1 -> ~5.1, 0.3 -> ~7.6.
+ """
if not direct_risks:
return 0.0
@@ -289,8 +303,7 @@ class PRBlastRadiusAnalyzer:
max_direct = max(r["risk_score"] for r in direct_risks)
breadth_bonus = min(len(transitive_affected) / 20.0, 1.0) # 0-1
- # Weighted: 40% avg, 40% max, 20% breadth — scaled to 10
- raw = (0.4 * avg_direct + 0.4 * max_direct + 0.2 * breadth_bonus)
- # Normalise pagerank-based scores (typically << 1) to 0-10
- score = min(raw * 100.0, 10.0)
+ combined = 0.5 * avg_direct + 0.5 * max_direct
+ file_term = 8.0 * (1.0 - math.exp(-10.0 * combined)) # 0-8, asymptotic
+ score = min(file_term + 2.0 * breadth_bonus, 10.0)
return round(score, 2)
diff --git a/packages/core/src/repowise/core/distill/filters/git_diff.py b/packages/core/src/repowise/core/distill/filters/git_diff.py
index 3cbbec2e7..4786d5162 100644
--- a/packages/core/src/repowise/core/distill/filters/git_diff.py
+++ b/packages/core/src/repowise/core/distill/filters/git_diff.py
@@ -11,16 +11,24 @@ from __future__ import annotations
import re
from typing import ClassVar
-from repowise.core.distill.filters.base import OutputFilter, cap_block
+from repowise.core.distill.filters.base import OutputFilter, cap_block, is_error_line
from repowise.core.distill.registry import filter_registry
_COMMAND_RE = re.compile(r"^git(?:\.exe)? (?:diff|show)\b")
_FILE_HEADER_RE = re.compile(r"^diff --git a/(?P<a>.+?) b/(?P<b>.+)$")
+# Diffstat row: " <path> | <count> +++---" or " <path> | Bin 120 -> 240 bytes".
+_STAT_LINE_RE = re.compile(r"^ (?P<path>.+?) +\| +(?:(?P<count>\d+)|Bin\b.*?)(?: +[+-]+)?\s*$")
+# Trailing roll-up: " 494 files changed, 30000 insertions(+), 12000 deletions(-)"
+_STAT_SUMMARY_RE = re.compile(
+ r"^\s*\d+ files? changed(?:, \d+ insertions?\(\+\))?(?:, \d+ deletions?\(-\))?\s*$"
+)
#: Files whose hunks are kept in full (the rest are summarized).
KEEP_FILES = 5
#: Per-file line cap for kept hunk blocks.
MAX_BLOCK_LINES = 160
+#: Stat-only mode: rows kept (ranked by churn); the rest collapse to a count.
+KEEP_STAT_FILES = 20
@filter_registry.register
@@ -64,6 +72,61 @@ class GitDiffFilter(OutputFilter):
return "\n".join(lines)
+@filter_registry.register
+class GitDiffStatFilter(OutputFilter):
+ """Stat-only ``git diff --stat`` / ``git show --stat`` rendering.
+
+ The hunk filter above deliberately skips ``--stat`` commands, and a
+ stat-only dump has no hunks to drop — so a 494-file diffstat used to
+ pass through raw. Keep the roll-up line plus the top rows by churn;
+ the long tail collapses to a count (the engine's marker preserves it).
+ """
+
+ name: ClassVar[str] = "git_diff_stat"
+ priority: ClassVar[int] = 11
+ min_lines: ClassVar[int] = 40
+
+ def matches_command(self, command: str) -> bool:
+ return bool(_COMMAND_RE.match(command)) and "--stat" in command
+
+ def matches_content(self, output: str) -> bool:
+ lines = [ln for ln in output.splitlines() if ln.strip()]
+ if not lines or any(_FILE_HEADER_RE.match(ln) for ln in lines[:5]):
+ return False
+ return bool(_STAT_SUMMARY_RE.match(lines[-1])) and bool(_STAT_LINE_RE.match(lines[0]))
+
+ def distill(self, output: str, *, command: str = "", exit_code: int = 0) -> str:
+ all_lines = output.splitlines()
+ if any(_FILE_HEADER_RE.match(ln) for ln in all_lines):
+ # Mixed --stat -p output: the hunk renderer's territory; fall
+ # back to raw rather than dropping hunks as "stat rows".
+ raise ValueError("diffstat mixed with hunks")
+
+ stat_rows: list[tuple[str, int]] = [] # (line, churn)
+ summary: str | None = None
+ extras: list[str] = [] # error lines and anything non-stat-shaped
+ for line in all_lines:
+ if not line.strip():
+ continue
+ if _STAT_SUMMARY_RE.match(line):
+ summary = line.strip()
+ elif m := _STAT_LINE_RE.match(line):
+ stat_rows.append((line, int(m.group("count") or 0)))
+ elif is_error_line(line):
+ extras.append(line)
+
+ if summary is None or len(stat_rows) <= KEEP_STAT_FILES:
+ raise ValueError("not a large stat-only diff")
+
+ top = sorted(stat_rows, key=lambda r: r[1], reverse=True)[:KEEP_STAT_FILES]
+ omitted = len(stat_rows) - len(top)
+ lines = [f"diff --stat: {summary}"]
+ lines.append(f"top {len(top)} files by churn ({omitted} more files omitted):")
+ lines.extend(ln for ln, _ in top)
+ lines.extend(extras)
+ return "\n".join(lines)
+
+
class _FileBlock:
__slots__ = ("added", "deleted", "lines", "path")
diff --git a/packages/core/src/repowise/core/distill/missed.py b/packages/core/src/repowise/core/distill/missed.py
index fd1b66c7c..a774b6e85 100644
--- a/packages/core/src/repowise/core/distill/missed.py
+++ b/packages/core/src/repowise/core/distill/missed.py
@@ -42,6 +42,7 @@ RATIO_FLOOR: dict[str, float] = {
"git_status": 0.15,
"git_log": 0.80,
"git_diff": 0.50,
+ "git_diff_stat": 0.50,
"search_results": 0.40,
"file_listing": 0.60,
"logs": 0.50,
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index 1ea45d192..341a84309 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -418,13 +418,21 @@ class _GenerationRun:
results = await asyncio.gather(*tasks)
# Embed the whole level in one batch before declaring it done — the
# next level's RAG search depends on these landing in the store.
- # Embedding is a RAG enhancement, not load-bearing, so failures are
- # swallowed at debug level.
+ # Embedding is a RAG enhancement, not load-bearing, so a failure must
+ # not abort generation — but it MUST be visible: a debug-level
+ # swallow here hid a 300k-token request rejection that silently lost
+ # every file-page embedding on init (`repowise reindex` repairs).
if embed_items and self.vector_store is not None:
try:
await self.vector_store.embed_batch(embed_items)
except Exception as e:
- log.debug("rag.embed_batch_failed", count=len(embed_items), error=str(e))
+ log.warning(
+ "rag.embed_batch_failed",
+ level=level,
+ count=len(embed_items),
+ error=str(e),
+ hint="semantic search will miss these pages; run `repowise reindex` to repair",
+ )
pages = [r for r in results if isinstance(r, GeneratedPage)]
if self.job_system is not None and self.job_id is not None:
for r in pages:
diff --git a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
index 58ed967f7..0fdd3187c 100644
--- a/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
+++ b/packages/core/src/repowise/core/ingestion/git_indexer/indexer.py
@@ -285,7 +285,9 @@ class GitIndexer:
)
return summary, results
- async def index_changed_files(self, changed_file_paths: list[str]) -> list[dict]:
+ async def index_changed_files(
+ self, changed_file_paths: list[str], all_files: set[str] | None = None
+ ) -> list[dict]:
"""Incremental update: re-index only changed files.
Mirrors ``index_repo``'s batched shape: one repo-wide commit-index
@@ -293,6 +295,12 @@ class GitIndexer:
subprocess per changed file), so the metadata an update produces is
identical to what a fresh full index would produce — including the
agent-provenance rollup, which the old per-file path never classified.
+
+ ``all_files`` is the repo's full tracked-file set. When provided (and
+ the tier includes co-change), the repo-wide co-change/entropy walk
+ re-runs so the changed files' partners stay fresh — without it,
+ ``index_file``'s empty defaults overwrite the init-computed partners
+ on every update, blanking exactly the files that change most.
"""
repo = self._get_repo()
if repo is None:
@@ -375,6 +383,34 @@ class GitIndexer:
except Exception as exc:
logger.debug("prior_defect_pass_failed", error=str(exc))
+ # Co-change partners + change entropy ride a repo-wide walk the
+ # per-file pass cannot produce. ``index_file`` resets both fields to
+ # empty defaults and the upsert overwrites rows field-by-field, so
+ # skipping this walk wiped the init-computed partners for every file
+ # an update touched. One ``git log --name-only`` subprocess, same as
+ # the full-index path.
+ if self.tier.includes_co_change and all_files:
+ try:
+ co_changes, change_entropy = await loop.run_in_executor(
+ None,
+ compute_co_changes_and_entropy,
+ repo,
+ set(all_files),
+ max(self.commit_limit, _DEFAULT_CO_CHANGE_COMMIT_LIMIT),
+ _DEFAULT_CO_CHANGE_MIN_COUNT,
+ None,
+ None,
+ as_of_ts,
+ )
+ for meta in results:
+ fp = meta["file_path"]
+ if fp in co_changes:
+ meta["co_change_partners_json"] = json.dumps(co_changes[fp])
+ if fp in change_entropy:
+ meta["change_entropy"] = change_entropy[fp]
+ except Exception as exc:
+ logger.debug("co_change_pass_failed", error=str(exc))
+
repo.close()
return results
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index fcc4d5445..a07d7abc2 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -709,10 +709,20 @@ def _parse_gitmodules(repo_root: Path) -> frozenset[str]:
def _load_gitignore_spec(repo_root: Path) -> pathspec.PathSpec:
- gitignore = repo_root / ".gitignore"
+ """Root ignore spec: ``.gitignore`` merged with ``.git/info/exclude``.
+
+ ``info/exclude`` is git's local-only ignore file — paths excluded there
+ (scratch dirs, private checkouts) are invisible to ``git status`` and
+ must be equally invisible to the index, or local-only files leak into
+ the graph, blast-radius lists, and generated docs.
+ """
lines: list[str] = []
- if gitignore.exists():
- lines = gitignore.read_text(encoding="utf-8", errors="ignore").splitlines()
+ for ignore_file in (
+ repo_root / ".gitignore",
+ repo_root / ".git" / "info" / "exclude",
+ ):
+ if ignore_file.exists():
+ lines.extend(ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines())
return pathspec.PathSpec.from_lines("gitwildmatch", lines)
diff --git a/packages/core/src/repowise/core/persistence/crud/git.py b/packages/core/src/repowise/core/persistence/crud/git.py
index 04877177b..94da588c5 100644
--- a/packages/core/src/repowise/core/persistence/crud/git.py
+++ b/packages/core/src/repowise/core/persistence/crud/git.py
@@ -93,10 +93,26 @@ async def get_all_git_metadata(session: AsyncSession, repository_id: str) -> dic
return {gm.file_path: gm for gm in result.scalars().all()}
+# Repo-wide-walk signals (co-change pairs, Hassan entropy). A pass that did
+# not run the walk — ESSENTIAL tier, or an incremental update without the
+# tracked-file set — reports the empty default for these; overwriting blindly
+# wiped the init-computed values for exactly the files that change most.
+_WALK_FIELD_EMPTIES: dict[str, tuple] = {
+ "co_change_partners_json": ("[]", "", None),
+ "change_entropy": (0, 0.0, None),
+}
+
+
def _update_git_metadata(existing: GitMetadata, meta: dict) -> None:
for key, val in meta.items():
- if key not in ("id", "repository_id") and hasattr(existing, key):
- setattr(existing, key, val)
+ if key in ("id", "repository_id") or not hasattr(existing, key):
+ continue
+ empties = _WALK_FIELD_EMPTIES.get(key)
+ if empties is not None and val in empties:
+ current = getattr(existing, key, None)
+ if current is not None and current not in empties:
+ continue # keep the previously computed signal
+ setattr(existing, key, val)
existing.updated_at = _now_utc()
diff --git a/packages/core/src/repowise/core/persistence/vector_store/_base.py b/packages/core/src/repowise/core/persistence/vector_store/_base.py
index 1bdf34f7b..7a2c83459 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/_base.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/_base.py
@@ -10,10 +10,39 @@ from __future__ import annotations
import math
from abc import ABC, abstractmethod
+from collections.abc import Iterator
from ..search import SearchResult
-__all__ = ["VectorStore", "cosine_similarity"]
+__all__ = [
+ "EMBED_BATCH_MAX_ITEMS",
+ "EMBED_TEXT_MAX_CHARS",
+ "VectorStore",
+ "cosine_similarity",
+ "iter_embed_chunks",
+]
+
+# One embedder call per chunk of this many items. OpenAI rejects embedding
+# requests past 300k total tokens — a generation level of 275 full wiki
+# pages (~560k tokens) failed in one giant request and silently lost the
+# whole level's embeddings (measured live: 400 max_tokens_per_request).
+# 16 items x EMBED_TEXT_MAX_CHARS worst-case is ~120k tokens, comfortably
+# under the cap and inside the embedder adapters' request timeouts
+# (a 16-page chunk measured at 0.6s against OpenAI).
+EMBED_BATCH_MAX_ITEMS = 16
+
+# Per-input cap (~7.5k tokens): embedding models reject a single input past
+# ~8,192 tokens, and one oversized page must not sink its whole chunk.
+EMBED_TEXT_MAX_CHARS = 30_000
+
+
+def iter_embed_chunks(
+ items: list[tuple[str, str, dict]],
+) -> Iterator[tuple[list[tuple[str, str, dict]], list[str]]]:
+ """Yield ``(chunk, capped_texts)`` slices sized for one embedder request."""
+ for start in range(0, len(items), EMBED_BATCH_MAX_ITEMS):
+ chunk = items[start : start + EMBED_BATCH_MAX_ITEMS]
+ yield chunk, [text[:EMBED_TEXT_MAX_CHARS] for _, text, _ in chunk]
def cosine_similarity(a: list[float], b: list[float]) -> float:
diff --git a/packages/core/src/repowise/core/persistence/vector_store/in_memory.py b/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
index b61ca9d2e..6cf1b49aa 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
@@ -5,7 +5,7 @@ from __future__ import annotations
from repowise.core.providers.embedding.base import Embedder
from ..search import SearchResult
-from ._base import VectorStore, cosine_similarity
+from ._base import VectorStore, cosine_similarity, iter_embed_chunks
__all__ = ["InMemoryVectorStore"]
@@ -27,12 +27,15 @@ class InMemoryVectorStore(VectorStore):
self._store[page_id] = (vectors[0], dict(metadata))
async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:
+ # Chunked like the persistent stores: this store is also the runtime
+ # fallback when LanceDB isn't installed, so it can sit in front of a
+ # real embedder with real request limits.
if not items:
return
- texts = [text for _, text, _ in items]
- vectors = await self._embedder.embed(texts)
- for (page_id, _text, metadata), vector in zip(items, vectors, strict=True):
- self._store[page_id] = (vector, dict(metadata))
+ for chunk, texts in iter_embed_chunks(items):
+ vectors = await self._embedder.embed(texts)
+ for (page_id, _text, metadata), vector in zip(chunk, vectors, strict=True):
+ self._store[page_id] = (vector, dict(metadata))
def _search_by_vector(self, q_vec: list[float], limit: int) -> list[SearchResult]:
scored: list[tuple[float, str, dict]] = []
diff --git a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
index 5e4866b68..582f91f4d 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
@@ -5,7 +5,7 @@ from __future__ import annotations
from repowise.core.providers.embedding.base import Embedder
from ..search import SearchResult
-from ._base import VectorStore
+from ._base import VectorStore, iter_embed_chunks
__all__ = ["LanceDBVectorStore"]
@@ -163,17 +163,35 @@ class LanceDBVectorStore(VectorStore):
await self._upsert_rows([self._row(page_id, vector, meta)])
async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:
+ """Embed and upsert in request-sized chunks with failure isolation.
+
+ One embedder call per :data:`EMBED_BATCH_MAX_ITEMS` items — a whole
+ generation level in a single request blew OpenAI's 300k-token cap
+ and silently lost every file-page embedding. A failed chunk no
+ longer sinks the rest; the summary error is raised at the end so
+ callers still see the loss.
+ """
if not items:
return
await self._ensure_connected()
- texts = [text for _, text, _ in items]
- vectors = await self._embedder.embed(texts)
- await self._ensure_table(vectors[0])
- rows = [
- self._row(page_id, vector, {"content": text, **metadata})
- for (page_id, text, metadata), vector in zip(items, vectors, strict=True)
- ]
- await self._upsert_rows(rows)
+ failed = 0
+ last_exc: Exception | None = None
+ for chunk, texts in iter_embed_chunks(items):
+ try:
+ vectors = await self._embedder.embed(texts)
+ await self._ensure_table(vectors[0])
+ rows = [
+ self._row(page_id, vector, {"content": text, **metadata})
+ for (page_id, text, metadata), vector in zip(chunk, vectors, strict=True)
+ ]
+ await self._upsert_rows(rows)
+ except Exception as exc: # isolate per chunk
+ failed += len(chunk)
+ last_exc = exc
+ if failed:
+ raise RuntimeError(
+ f"embed_batch: {failed}/{len(items)} items failed to embed"
+ ) from last_exc
async def _search_by_vector(self, q_vec: list[float], limit: int) -> list[SearchResult]:
# Query with explicit cosine distance so ``_distance`` is a cosine
diff --git a/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py b/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
index 065f95274..325e21a96 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
from repowise.core.providers.embedding.base import Embedder
from ..search import SearchResult
-from ._base import VectorStore
+from ._base import VectorStore, iter_embed_chunks
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -73,21 +73,22 @@ class PgVectorStore(VectorStore):
async def embed_batch(self, items: list[tuple[str, str, dict]]) -> None:
if not items:
return
- texts = [text for _, text, _ in items]
- vectors = await self._embedder.embed(texts)
- params = [
- {"emb": _encode(vector), "pid": page_id}
- for (page_id, _text, _meta), vector in zip(items, vectors, strict=True)
- ]
from sqlalchemy.sql import text as sa_text
stmt = sa_text("UPDATE wiki_pages SET embedding = CAST(:emb AS vector) WHERE id = :pid")
- async with self._session_factory() as session:
- # executemany: one driver round-trip batch instead of one UPDATE
- # round-trip per row.
- await session.execute(stmt, params)
- await session.commit()
+ # Chunked: one embedder request per slice (a whole generation level
+ # in one request blew OpenAI's 300k-token cap), then one executemany
+ # round-trip per slice.
+ for chunk, texts in iter_embed_chunks(items):
+ vectors = await self._embedder.embed(texts)
+ params = [
+ {"emb": _encode(vector), "pid": page_id}
+ for (page_id, _text, _meta), vector in zip(chunk, vectors, strict=True)
+ ]
+ async with self._session_factory() as session:
+ await session.execute(stmt, params)
+ await session.commit()
async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
q_vecs = await self._embedder.embed([query])
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.py
index aa5a0b75d..3f5182fe0 100644
--- a/packages/core/src/repowise/core/pipeline/incremental.py
+++ b/packages/core/src/repowise/core/pipeline/incremental.py
@@ -199,7 +199,11 @@ async def rebuild_graph_and_git(
tier=tier,
)
changed_paths = build_filtered_changed_paths(file_diffs, exclude_patterns)
- updated_meta = await git_indexer.index_changed_files(changed_paths)
+ # The full tracked-file set lets the indexer re-run the repo-wide
+ # co-change walk so partners aren't wiped to "[]" for changed files.
+ updated_meta = await git_indexer.index_changed_files(
+ changed_paths, all_files=set(source_map.keys())
+ )
git_meta_map = {m["file_path"]: m for m in updated_meta}
graph_builder.update_co_change_edges(git_meta_map)
except Exception as exc:
diff --git a/packages/server/src/repowise/server/mcp_server/_helpers.py b/packages/server/src/repowise/server/mcp_server/_helpers.py
index 4dee2e33c..282baa992 100644
--- a/packages/server/src/repowise/server/mcp_server/_helpers.py
+++ b/packages/server/src/repowise/server/mcp_server/_helpers.py
@@ -431,12 +431,30 @@ def _compute_alignment(
def _get_exclude_spec(repo_path: "Path | str") -> "Any":
- """Compile the repo's ``exclude_patterns`` into a PathSpec, or None."""
+ """Compile the repo's exclusion rules into a PathSpec, or None.
+
+ Unions ``.repowise/config.yaml`` ``exclude_patterns`` with the repo's
+ gitignore stack (``.gitignore`` + ``.git/info/exclude``). Indexes built
+ before the traverser honoured ``info/exclude`` still contain rows for
+ local-only scratch dirs; filtering them at query time keeps those paths
+ out of tool responses without forcing a reindex.
+ """
+ from pathlib import Path
+
import pathspec
from repowise.core.repo_config import load_repo_config
- patterns = load_repo_config(repo_path).get("exclude_patterns") or []
+ patterns = list(load_repo_config(repo_path).get("exclude_patterns") or [])
+ root = Path(repo_path)
+ for ignore_file in (root / ".gitignore", root / ".git" / "info" / "exclude"):
+ try:
+ if ignore_file.exists():
+ patterns.extend(
+ ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines()
+ )
+ except OSError:
+ continue
if not patterns:
return None
return pathspec.PathSpec.from_lines("gitwildmatch", patterns)
diff --git a/packages/server/src/repowise/server/mcp_server/tool_context/context.py b/packages/server/src/repowise/server/mcp_server/tool_context/context.py
index 894935991..e269a53bc 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_context/context.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_context/context.py
@@ -76,6 +76,11 @@ async def get_context(
bodies of the highest-PageRank symbols inlined under a
token budget. A fraction of the cost of Read for
structure-level questions (file targets only).
+ DEFAULT for file targets above ~80 lines: the skeleton
+ replaces the bare symbol list (strictly better per
+ token). Pass compact=False for the symbol-list card.
+ A ``mostly_full`` flag marks files where a direct Read
+ costs little more.
Example: get_context(["src/auth/service.py", "src/auth/middleware.py"])
Example: get_context(["src/auth/service.py::verify_token"], include=["callers"])
@@ -83,7 +88,8 @@ async def get_context(
Args:
targets: file paths, module paths, or qualified symbol IDs.
include: list of optional data blocks (defaults are always returned).
- compact: default True (signatures only). False adds structure+imports+docstrings.
+ compact: default True (signatures only; large file targets auto-upgrade
+ to skeleton). False adds structure+imports+docstrings instead.
repo: usually omitted.
"""
if repo == "all":
diff --git a/packages/server/src/repowise/server/mcp_server/tool_context/enrichment.py b/packages/server/src/repowise/server/mcp_server/tool_context/enrichment.py
index 5fef6ef4b..26fee7162 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_context/enrichment.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_context/enrichment.py
@@ -431,3 +431,12 @@ async def _resolve_skeleton(
result_data["skeleton"]["note"] = (
"No usable symbol bounds for this file — returned source as-is."
)
+ elif result.pct_of_full > 40.0:
+ # Small files skeletonize poorly: when the skeleton is already a
+ # large fraction of the source, tell the agent a Read costs little
+ # more and carries everything.
+ result_data["skeleton"]["mostly_full"] = True
+ result_data["skeleton"]["note"] = (
+ f"Skeleton is {round(result.pct_of_full, 1)}% of the full file — "
+ "a direct Read costs little more."
+ )
diff --git a/packages/server/src/repowise/server/mcp_server/tool_context/targets.py b/packages/server/src/repowise/server/mcp_server/tool_context/targets.py
index 861ebbe68..f7b6f1416 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_context/targets.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_context/targets.py
@@ -46,6 +46,15 @@ from repowise.server.mcp_server.tool_context.kg import (
)
+# Skeleton-by-default threshold for file targets. Measured on this repo: a
+# 1,400-line file's default card costs ~2.5k tokens for 16 bare signatures,
+# while the smart skeleton costs ~1.7k and carries every signature plus
+# docstrings and the highest-PageRank bodies — strictly better per token.
+# Small files skeletonize poorly (pct_of_full approaches a plain Read), so
+# the card remains the default below this line count.
+_SKELETON_AUTO_MIN_LINES = 80
+
+
def _escape_like(value: str) -> str:
"""Escape LIKE metacharacters (``%``, ``_``) and the escape char itself.
@@ -320,6 +329,9 @@ async def _resolve_one_target(
result_data["target"] = target
result_data["type"] = target_type
+ want_skeleton = bool(include and "skeleton" in include)
+ auto_skeleton = False
+
# --- Docs ---
# "full_doc" implies "docs" — entering the docs block whenever either is requested.
if include is None or "docs" in include or "full_doc" in include:
@@ -343,7 +355,20 @@ async def _resolve_one_target(
symbols = res.scalars().all()
classes = [s.name for s in symbols if s.kind == "class"]
functions = [s.name for s in symbols if s.kind in ("function", "method")]
- if include and "skeleton" in include:
+ # Skeleton-by-default: for file targets of meaningful size the
+ # smart skeleton dominates the bare signature list per token, so
+ # the default card upgrades itself. compact=False (the rich
+ # symbol card) and full_doc both opt out.
+ if not want_skeleton and compact and not want_full_doc:
+ total_loc = max((s.end_line or 0 for s in symbols), default=0)
+ if total_loc > _SKELETON_AUTO_MIN_LINES:
+ want_skeleton = auto_skeleton = True
+ # Explicitly requested skeleton suppresses the symbol list up
+ # front; the auto default still builds the card and only swaps
+ # it out once the skeleton actually resolved (see bottom), so a
+ # moved/unreadable source file degrades to the card, not to an
+ # error-only response.
+ if want_skeleton and not auto_skeleton:
# The skeleton block already renders every signature with
# line bounds — repeating the symbol list in docs would
# roughly double the response for zero information. Keep
@@ -754,10 +779,31 @@ async def _resolve_one_target(
if include and "health" in include:
await _resolve_health(session, repository, target, target_type, result_data)
- # --- Skeleton (distill) ---
- if include and "skeleton" in include:
+ # --- Skeleton (distill) — explicit include or the file-target default ---
+ if want_skeleton:
await _resolve_skeleton(
session, repository, target, target_type, result_data, repo_root=repo_root
)
+ skeleton = result_data.get("skeleton")
+ if auto_skeleton and isinstance(skeleton, dict):
+ if "error" in skeleton:
+ # Auto-upgrade failed (source moved/unreadable) — keep the
+ # symbol card the docs block already built and drop the
+ # failed block so the default response stays usable.
+ result_data.pop("skeleton", None)
+ else:
+ skeleton["auto"] = True
+ skeleton["opt_out_hint"] = (
+ "Skeleton is the default for file targets above "
+ f"{_SKELETON_AUTO_MIN_LINES} lines. Pass compact=False "
+ "for the symbol-list card instead."
+ )
+ docs_block = result_data.get("docs")
+ if isinstance(docs_block, dict):
+ # The skeleton renders every signature with line bounds —
+ # the symbol list would double the response for zero
+ # information.
+ docs_block.pop("symbols", None)
+ docs_block.pop("symbols_truncated", None)
return result_data
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.py
index 370e5ba54..d6341a468 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_search.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_search.py
@@ -46,6 +46,49 @@ def _looks_like_exact_token(query: str) -> bool:
return bool(_IDENT_QUERY_RE.match(stripped))
+# Decision records are short, dense title-statements; they win cosine
+# similarity against long file-page embeddings on any query containing
+# design nouns ("store", "SQLite", "cap", "prune") and crowd file pages
+# out of the top ranks entirely. Down-weight them unless the query is
+# why-shaped — rationale questions are get_why's territory, but a caller
+# who phrases one here clearly wants the decision pages ranked honestly.
+_DECISION_DOWNWEIGHT = 0.6
+
+_WHY_SHAPED_RE = re.compile(
+ r"^\s*(why|when\s+did|when\s+was|who\s+decided|who\s+chose|what\s+was\s+the\s+(reason|rationale))\b"
+ r"|\b(decision|decided|rationale|adr)\b",
+ re.IGNORECASE,
+)
+
+
+def _is_why_shaped(query: str) -> bool:
+ """True when the query asks for rationale, so decision records should rank naturally."""
+ return bool(_WHY_SHAPED_RE.search(query))
+
+
+def _downweight_decisions(output: list[dict], query: str) -> None:
+ """Scale decision_record relevance in place unless the query is why-shaped."""
+ if _is_why_shaped(query):
+ return
+ for item in output:
+ if item.get("page_type") == "decision_record" and item.get("relevance_score"):
+ item["relevance_score"] = round(
+ item["relevance_score"] * _DECISION_DOWNWEIGHT, 4
+ )
+
+
+def _fetch_limit_for(limit: int, kind: str | None) -> int:
+ """Over-fetch headroom for post-filters and decision down-weighting.
+
+ Always over-fetch at least 3x: without headroom the down-weighting can
+ only reorder a window that decision records may already fill, so file
+ pages never surface. ``kind`` trims hardest (decision/module/overview
+ pages all classify as "doc"), so it gets 6x — 3x was measured to leave
+ zero implementation pages in the window on decision-heavy queries.
+ """
+ return limit * (6 if kind else 3)
+
+
# Path-prefix heuristics for the ``kind`` filter. We classify a hit's
# target_path against these prefixes; if none match, the hit falls into
# ``other`` and is dropped only when the caller asked for a specific kind.
@@ -79,7 +122,7 @@ def _classify_hit_kind(target_path: str, page_type: str) -> str:
async def _search_single_repo(
- ctx, query: str, limit: int, page_type: str | None
+ ctx, query: str, limit: int, page_type: str | None, kind: str | None = None
) -> tuple[list[dict], str]:
"""Run search against a single repo context.
@@ -88,13 +131,17 @@ async def _search_single_repo(
the list — embedding misses fall through to FTS silently in the existing
code, and the agent has no way to distinguish a strong embedding hit
from a fallback BM25 hit otherwise.
+
+ The ``kind`` filter runs here, on the over-fetched list and BEFORE the
+ limit cut — filtering after per-repo truncation returned fewer than
+ ``limit`` results (frequently zero) in the federated path.
"""
# Wait for vector store readiness
if ctx.vector_store_ready is not None:
with contextlib.suppress(TimeoutError):
await asyncio.wait_for(ctx.vector_store_ready.wait(), timeout=30.0)
- fetch_limit = limit * 3 if page_type else limit
+ fetch_limit = _fetch_limit_for(limit, kind)
results = []
method = "embedding"
with contextlib.suppress(TimeoutError, Exception):
@@ -121,10 +168,12 @@ async def _search_single_repo(
"relevance_score": r.score,
})
- output = output[:limit]
+ _downweight_decisions(output, query)
+ output.sort(key=lambda x: x.get("relevance_score", 0), reverse=True)
# Attach target_path and drop excluded hits per repo (so federated search
- # honours each repo's own exclude_patterns) before aggregation.
+ # honours each repo's own exclude_patterns) before aggregation. Runs on
+ # the over-fetched list so the kind filter below has real headroom.
if output:
page_ids = [item["page_id"] for item in output]
async with get_session(ctx.session_factory) as session:
@@ -136,16 +185,24 @@ async def _search_single_repo(
item["target_path"] = page_info.get(item["page_id"], "")
output = filter_dicts_by_key(output, "target_path", _get_exclude_spec(ctx.path))
- return output, method
+ if kind:
+ output = [
+ item for item in output
+ if _classify_hit_kind(item.get("target_path", ""), item.get("page_type", "")) == kind
+ ]
+
+ return output[:limit], method
-async def _federated_search(query: str, limit: int, page_type: str | None) -> dict:
+async def _federated_search(
+ query: str, limit: int, page_type: str | None, kind: str | None = None
+) -> dict:
"""Search across all repos using Reciprocal Rank Fusion."""
contexts = await _resolve_all_contexts()
all_results = []
for ctx in contexts:
- repo_results, repo_method = await _search_single_repo(ctx, query, limit, page_type)
+ repo_results, repo_method = await _search_single_repo(ctx, query, limit, page_type, kind)
for rank, item in enumerate(repo_results):
item["repo"] = ctx.alias
item["rrf_score"] = 1.0 / (rank + 60) # RRF constant k=60
@@ -205,12 +262,9 @@ async def search_codebase(
)
if repo == "all":
- federated = await _federated_search(query, limit, page_type)
- if kind:
- federated["results"] = [
- r for r in federated["results"]
- if _classify_hit_kind(r.get("target_path", ""), r.get("page_type", "")) == kind
- ]
+ # kind is filtered per-repo inside _search_single_repo, before each
+ # repo's limit cut, so the fused list is already kind-pure and full.
+ federated = await _federated_search(query, limit, page_type, kind)
if grep_hint:
federated["grep_hint"] = grep_hint
return federated
@@ -229,10 +283,9 @@ async def search_codebase(
# Try semantic search, fall back to FTS. Track which backend supplied the
# hits so the response can surface it per-result — silent fallback hides
# a quality cliff that the agent should weigh into its trust budget.
- # Over-fetch whenever a post-filter (page_type or kind) will trim hits;
- # without the head-room a kind filter applied to an already-truncated
- # list returned fewer than ``limit`` results — frequently zero.
- fetch_limit = limit * 3 if (page_type or kind) else limit
+ # Always over-fetch (see _fetch_limit_for): post-filters trim hits, and
+ # decision down-weighting needs file pages inside the window to promote.
+ fetch_limit = _fetch_limit_for(limit, kind)
results = []
search_method = "embedding"
with contextlib.suppress(TimeoutError, Exception):
@@ -262,6 +315,8 @@ async def search_codebase(
}
)
+ _downweight_decisions(output, query)
+
# Batch-lookup page target paths for the kind filter + git freshness boost
if output:
page_ids = [item["page_id"] for item in output]
diff --git a/tests/unit/analysis/test_pr_blast_risk.py b/tests/unit/analysis/test_pr_blast_risk.py
new file mode 100644
index 000000000..71f40a0cb
--- /dev/null
+++ b/tests/unit/analysis/test_pr_blast_risk.py
@@ -0,0 +1,65 @@
+"""Calibration of PRBlastRadiusAnalyzer._compute_overall_risk.
+
+Regression: the old ``min(raw * 100, 10)`` normalisation scaled the 0-1
+breadth bonus to 0-20 points, so any PR with >=20 transitive dependents —
+i.e. any PR touching a hotspot — reported exactly 10.0. The score must
+discriminate between small clean diffs, moderate diffs, and hotspot-heavy
+diffs, and only approach 10 asymptotically.
+"""
+
+from __future__ import annotations
+
+from repowise.core.analysis.pr_blast import PRBlastRadiusAnalyzer
+
+
+def _direct(*scores: float) -> list[dict]:
+ return [{"risk_score": s} for s in scores]
+
+
+def _transitive(n: int) -> list[dict]:
+ return [{"path": f"f{i}.py"} for i in range(n)]
+
+
+class TestOverallRiskCalibration:
+ def test_empty_is_zero(self):
+ assert PRBlastRadiusAnalyzer._compute_overall_risk([], []) == 0.0
+
+ def test_small_clean_diff_scores_low(self):
+ # Three low-centrality, low-churn files, narrow blast.
+ score = PRBlastRadiusAnalyzer._compute_overall_risk(
+ _direct(0.002, 0.001, 0.003), _transitive(2)
+ )
+ assert 0.0 < score < 2.0
+
+ def test_hotspot_diff_does_not_saturate(self):
+ # One genuine hotspot (pagerank*(1+temporal) ~ 0.2) plus wide blast:
+ # previously this pinned at exactly 10.0.
+ score = PRBlastRadiusAnalyzer._compute_overall_risk(
+ _direct(0.2, 0.01, 0.005), _transitive(40)
+ )
+ assert score < 10.0
+ assert score > 3.0
+
+ def test_breadth_alone_cannot_max_the_score(self):
+ # Regression core: >=20 transitive dependents alone used to add 20
+ # points. Breadth now contributes at most 2.
+ score = PRBlastRadiusAnalyzer._compute_overall_risk(_direct(0.001), _transitive(100))
+ assert score < 4.0
+
+ def test_monotonic_in_file_risk(self):
+ low = PRBlastRadiusAnalyzer._compute_overall_risk(_direct(0.01), _transitive(5))
+ mid = PRBlastRadiusAnalyzer._compute_overall_risk(_direct(0.05), _transitive(5))
+ high = PRBlastRadiusAnalyzer._compute_overall_risk(_direct(0.3), _transitive(5))
+ assert low < mid < high
+
+ def test_monotonic_in_breadth(self):
+ narrow = PRBlastRadiusAnalyzer._compute_overall_risk(_direct(0.05), _transitive(0))
+ wide = PRBlastRadiusAnalyzer._compute_overall_risk(_direct(0.05), _transitive(40))
+ assert narrow < wide
+ assert round(wide - narrow, 6) <= 2.0 # breadth term is capped at 2 points
+
+ def test_never_exceeds_ten(self):
+ score = PRBlastRadiusAnalyzer._compute_overall_risk(
+ _direct(5.0, 5.0, 5.0), _transitive(500)
+ )
+ assert score <= 10.0
diff --git a/tests/unit/distill/test_filters.py b/tests/unit/distill/test_filters.py
index 2f6649423..df958aaf1 100644
--- a/tests/unit/distill/test_filters.py
+++ b/tests/unit/distill/test_filters.py
@@ -294,6 +294,65 @@ def test_git_diff_unrecognized_raises() -> None:
f.distill("random text\n" * 50, command="git diff")
+# -- git_diff_stat ------------------------------------------------------------------
+
+
+def _stat_output(n_files: int = 100) -> str:
+ lines = []
+ for i in range(n_files):
+ churn = (i * 7) % 250 + 1
+ bars = "+" * min(churn, 20)
+ lines.append(f" pkg/module_{i:03d}.py | {churn:4d} {bars}")
+ lines.append(" img/logo.png | Bin 1024 -> 2048 bytes")
+ lines.append(f" {n_files + 1} files changed, 12345 insertions(+), 6789 deletions(-)")
+ return "\n".join(lines) + "\n"
+
+
+def test_git_diff_stat_keeps_top_by_churn() -> None:
+ f = filter_registry.get("git_diff_stat")
+ assert f is not None
+ raw = _stat_output(100)
+ distilled = f.distill(raw, command="git diff --stat HEAD~50..HEAD")
+ head = distilled.splitlines()
+ assert head[0].startswith("diff --stat: 101 files changed")
+ assert "81 more files omitted" in head[1] # 101 rows - 20 kept
+ # The highest-churn row survives; a tiny row does not.
+ max_line = max(
+ (ln for ln in raw.splitlines() if "|" in ln and "Bin" not in ln and "changed" not in ln),
+ key=lambda ln: int(ln.split("|")[1].strip().split()[0]),
+ )
+ assert max_line in distilled
+ assert _pct(raw, distilled) >= 50.0
+
+
+def test_git_diff_stat_command_routing() -> None:
+ stat = filter_registry.get("git_diff_stat")
+ hunks = filter_registry.get("git_diff")
+ assert stat.matches_command("git diff --stat HEAD~5")
+ assert not stat.matches_command("git diff HEAD~5")
+ assert not hunks.matches_command("git diff --stat HEAD~5")
+
+
+def test_git_diff_stat_content_sniff() -> None:
+ stat = filter_registry.get("git_diff_stat")
+ assert stat.matches_content(_stat_output(50))
+ assert not stat.matches_content("random text\n" * 50)
+
+
+def test_git_diff_stat_mixed_with_hunks_raises() -> None:
+ f = filter_registry.get("git_diff_stat")
+ mixed = _stat_output(60) + "diff --git a/x.py b/x.py\n+++ b/x.py\n+new line\n"
+ with pytest.raises(ValueError):
+ f.distill(mixed, command="git diff --stat -p")
+
+
+def test_git_diff_stat_small_diff_raises() -> None:
+ # <= KEEP_STAT_FILES rows: nothing to save, fall back to raw.
+ f = filter_registry.get("git_diff_stat")
+ with pytest.raises(ValueError):
+ f.distill(_stat_output(10), command="git diff --stat")
+
+
# -- file_listing -------------------------------------------------------------------
diff --git a/tests/unit/ingestion/test_git_update_equivalence.py b/tests/unit/ingestion/test_git_update_equivalence.py
index bd3a908f0..509d00533 100644
--- a/tests/unit/ingestion/test_git_update_equivalence.py
+++ b/tests/unit/ingestion/test_git_update_equivalence.py
@@ -108,6 +108,53 @@ async def test_update_path_classifies_agent_provenance(tmp_path) -> None:
assert meta["agent_tier_counts_json"] != "{}"
+def _build_repo_with_cochange(tmp_path):
+ """Repo where a.py and b.py change together three times (recent)."""
+ import git as gitpython
+
+ repo = gitpython.Repo.init(tmp_path)
+ with repo.config_writer() as cw:
+ cw.set_value("user", "name", "Alice")
+ cw.set_value("user", "email", "[email protected]")
+
+ for i in range(3):
+ (tmp_path / "a.py").write_text(f"x = {i}\n")
+ (tmp_path / "b.py").write_text(f"y = {i}\n")
+ repo.index.add(["a.py", "b.py"])
+ repo.index.commit(f"feat: joint change {i} touching both modules")
+ repo.close()
+
+
+async def test_update_path_recomputes_co_change_partners(tmp_path) -> None:
+ """Regression: index_changed_files left co_change_partners_json at its
+ empty default, and the field-by-field upsert then wiped the partners
+ computed at init for every file an update touched — i.e. the hotspots."""
+ import json
+
+ _build_repo_with_cochange(tmp_path)
+
+ upd = GitIndexer(tmp_path, tier=GitIndexTier.FULL)
+ upd_meta = await upd.index_changed_files(["a.py"], all_files={"a.py", "b.py"})
+ (meta,) = upd_meta
+
+ partners = json.loads(meta["co_change_partners_json"])
+ assert partners, "co-change walk must repopulate partners on update"
+ assert partners[0]["file_path"] == "b.py"
+ assert meta["change_entropy"] > 0.0
+
+
+async def test_update_path_essential_tier_skips_co_change_walk(tmp_path) -> None:
+ _build_repo_with_cochange(tmp_path)
+
+ upd = GitIndexer(tmp_path, tier=GitIndexTier.ESSENTIAL)
+ upd_meta = await upd.index_changed_files(["a.py"], all_files={"a.py", "b.py"})
+ (meta,) = upd_meta
+
+ # ESSENTIAL defers the walk; the upsert guard preserves any existing DB
+ # value, so the metadata dict itself stays at the empty default.
+ assert meta["co_change_partners_json"] == "[]"
+
+
def test_thread_repo_pool_reuses_per_thread_and_closes(tmp_path) -> None:
_build_repo(tmp_path)
indexer = GitIndexer(tmp_path)
diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.py
index 881b5debb..f468c293d 100644
--- a/tests/unit/ingestion/test_traverser.py
+++ b/tests/unit/ingestion/test_traverser.py
@@ -121,6 +121,23 @@ class TestFileTraverser:
assert not any("debug.log" in p for p in paths)
assert not any("secret" in p for p in paths)
+ def test_respects_git_info_exclude(self, tmp_path: Path) -> None:
+ # .git/info/exclude is git's local-only ignore file — scratch dirs
+ # excluded there are invisible to git status and must be equally
+ # invisible to the index (they leaked into blast-radius lists).
+ info = tmp_path / ".git" / "info"
+ info.mkdir(parents=True)
+ (info / "exclude").write_text("local-stash/\n*.scratch\n")
+ (tmp_path / "app.py").write_text("pass")
+ (tmp_path / "notes.scratch").write_text("pass")
+ (tmp_path / "local-stash").mkdir()
+ (tmp_path / "local-stash" / "probe.py").write_text("pass")
+ traverser = FileTraverser(tmp_path)
+ paths = [f.path for f in traverser.traverse()]
+ assert any("app.py" in p for p in paths)
+ assert not any("local-stash" in p for p in paths)
+ assert not any("notes.scratch" in p for p in paths)
+
def test_skips_oversized_files(self, tmp_path: Path) -> None:
big = tmp_path / "big.py"
big.write_bytes(b"x = 1\n" * 200_000) # ~1.2 MB
diff --git a/tests/unit/persistence/test_git_meta_upsert_guard.py b/tests/unit/persistence/test_git_meta_upsert_guard.py
new file mode 100644
index 000000000..433539304
--- /dev/null
+++ b/tests/unit/persistence/test_git_meta_upsert_guard.py
@@ -0,0 +1,49 @@
+"""The git-metadata upsert must not wipe repo-wide-walk signals.
+
+``co_change_partners_json`` and ``change_entropy`` are produced by the
+repo-wide co-change walk. A pass that did not run the walk (ESSENTIAL tier,
+legacy incremental updates) reports the empty default for both — blindly
+overwriting blanked the init-computed values for exactly the files that
+change most.
+"""
+
+from __future__ import annotations
+
+from repowise.core.persistence.crud.git import _update_git_metadata
+from repowise.core.persistence.models import GitMetadata
+
+
+def _existing(**kwargs) -> GitMetadata:
+ return GitMetadata(file_path="src/a.py", **kwargs)
+
+
+class TestWalkFieldPreservation:
+ def test_empty_partners_do_not_overwrite(self):
+ existing = _existing(co_change_partners_json='[{"file_path": "src/b.py"}]')
+ _update_git_metadata(existing, {"co_change_partners_json": "[]"})
+ assert existing.co_change_partners_json == '[{"file_path": "src/b.py"}]'
+
+ def test_fresh_partners_do_overwrite(self):
+ existing = _existing(co_change_partners_json='[{"file_path": "src/b.py"}]')
+ _update_git_metadata(existing, {"co_change_partners_json": '[{"file_path": "src/c.py"}]'})
+ assert existing.co_change_partners_json == '[{"file_path": "src/c.py"}]'
+
+ def test_empty_over_empty_is_fine(self):
+ existing = _existing(co_change_partners_json="[]")
+ _update_git_metadata(existing, {"co_change_partners_json": "[]"})
+ assert existing.co_change_partners_json == "[]"
+
+ def test_zero_entropy_does_not_overwrite(self):
+ existing = _existing(change_entropy=1.37)
+ _update_git_metadata(existing, {"change_entropy": 0.0})
+ assert existing.change_entropy == 1.37
+
+ def test_real_entropy_does_overwrite(self):
+ existing = _existing(change_entropy=1.37)
+ _update_git_metadata(existing, {"change_entropy": 2.5})
+ assert existing.change_entropy == 2.5
+
+ def test_other_fields_still_overwrite_normally(self):
+ existing = _existing(commit_count_total=5)
+ _update_git_metadata(existing, {"commit_count_total": 0})
+ assert existing.commit_count_total == 0
diff --git a/tests/unit/persistence/test_vector_store.py b/tests/unit/persistence/test_vector_store.py
index 6c42cc147..c0e47f66d 100644
--- a/tests/unit/persistence/test_vector_store.py
+++ b/tests/unit/persistence/test_vector_store.py
@@ -407,3 +407,75 @@ async def test_lancedb_reindex_recreates_table_on_dimension_change(tmp_path, moc
assert {r.page_id for r in results} == {"p2"}
finally:
await store.close()
+
+
+# ---------------------------------------------------------------------------
+# embed_batch chunking (regression: one giant request lost a whole level)
+# ---------------------------------------------------------------------------
+
+
+class _RecordingEmbedder:
+ """Mock-dim embedder that records every embed() call's batch shape."""
+
+ dimensions = 8
+
+ def __init__(self, fail_on_call: int | None = None) -> None:
+ self.calls: list[list[str]] = []
+ self._fail_on_call = fail_on_call
+
+ async def embed(self, texts: list[str]) -> list[list[float]]:
+ self.calls.append(list(texts))
+ if self._fail_on_call is not None and len(self.calls) == self._fail_on_call:
+ raise RuntimeError("simulated 400 max_tokens_per_request")
+ return [[1.0] + [0.0] * 7 for _ in texts]
+
+
+def _items(n: int, text: str = "content") -> list[tuple[str, str, dict]]:
+ return [(f"p{i}", text, {"target_path": f"f{i}.py"}) for i in range(n)]
+
+
+async def test_in_memory_embed_batch_chunks_requests():
+ # Regression: a generation level of 275 full pages went out as ONE
+ # embedder request (~560k tokens) and failed OpenAI's 300k cap, silently
+ # losing every file-page embedding. Batches must be chunked.
+ from repowise.core.persistence.vector_store import InMemoryVectorStore
+ from repowise.core.persistence.vector_store._base import EMBED_BATCH_MAX_ITEMS
+
+ emb = _RecordingEmbedder()
+ store = InMemoryVectorStore(emb)
+ await store.embed_batch(_items(EMBED_BATCH_MAX_ITEMS * 2 + 3))
+ assert len(emb.calls) == 3
+ assert all(len(c) <= EMBED_BATCH_MAX_ITEMS for c in emb.calls)
+ assert len(await store.list_page_ids()) == EMBED_BATCH_MAX_ITEMS * 2 + 3
+
+
+async def test_in_memory_embed_batch_caps_text_length():
+ from repowise.core.persistence.vector_store import InMemoryVectorStore
+ from repowise.core.persistence.vector_store._base import EMBED_TEXT_MAX_CHARS
+
+ emb = _RecordingEmbedder()
+ store = InMemoryVectorStore(emb)
+ await store.embed_batch(_items(1, text="x" * (EMBED_TEXT_MAX_CHARS + 5_000)))
+ assert len(emb.calls[0][0]) == EMBED_TEXT_MAX_CHARS
+
+
+async def test_lancedb_embed_batch_isolates_failed_chunk(tmp_path):
+ # One failing chunk must not sink the rest of the level, but the loss
+ # must surface as an error (the old code swallowed it entirely).
+ pytest.importorskip("lancedb")
+ from repowise.core.persistence.vector_store import LanceDBVectorStore
+ from repowise.core.persistence.vector_store._base import EMBED_BATCH_MAX_ITEMS
+
+ emb = _RecordingEmbedder(fail_on_call=2)
+ store = LanceDBVectorStore(str(tmp_path / "lance"), emb)
+ try:
+ with pytest.raises(RuntimeError, match="failed to embed"):
+ await store.embed_batch(_items(EMBED_BATCH_MAX_ITEMS * 3))
+ ids = await store.list_page_ids()
+ # Chunks 1 and 3 persisted; only chunk 2 was lost.
+ assert len(ids) == EMBED_BATCH_MAX_ITEMS * 2
+ finally:
+ await store.close()
diff --git a/tests/unit/server/mcp/test_context_skeleton.py b/tests/unit/server/mcp/test_context_skeleton.py
index ae4b431d7..817f0647b 100644
--- a/tests/unit/server/mcp/test_context_skeleton.py
+++ b/tests/unit/server/mcp/test_context_skeleton.py
@@ -64,10 +64,59 @@ async def test_skeleton_missing_source_file(setup_mcp, tmp_path, monkeypatch):
@pytest.mark.asyncio
-async def test_skeleton_not_included_by_default(setup_mcp, tmp_path, monkeypatch):
+async def test_skeleton_is_default_for_large_file_targets(setup_mcp, tmp_path, monkeypatch):
+ # service.py spans 100 lines (> the 80-line threshold): the default card
+ # auto-upgrades to the skeleton and drops the redundant symbol list.
from repowise.server.mcp_server import _state, get_context
_write_source(tmp_path)
monkeypatch.setattr(_state, "_repo_path", str(tmp_path))
result = await get_context(["src/auth/service.py"])
- assert "skeleton" not in result["targets"]["src/auth/service.py"]
+ card = result["targets"]["src/auth/service.py"]
+ sk = card["skeleton"]
+ assert sk["auto"] is True
+ assert "compact=False" in sk["opt_out_hint"]
+ assert "class AuthService:" in sk["text"]
+ assert "symbols" not in card["docs"]
+ # Summary and freshness still ride along.
+ assert card["docs"].get("summary") is not None
+ assert "freshness" in card
+
+
+async def test_small_file_keeps_symbol_card(setup_mcp, tmp_path, monkeypatch):
+ # models.py's symbols end at line 30 — below the threshold, no skeleton.
+ from repowise.server.mcp_server import _state, get_context
+
+ monkeypatch.setattr(_state, "_repo_path", str(tmp_path))
+ result = await get_context(["src/db/models.py"])
+ card = result["targets"]["src/db/models.py"]
+ assert "skeleton" not in card
+ assert card["docs"]["symbols"]
+
+
+async def test_compact_false_opts_out_of_auto_skeleton(setup_mcp, tmp_path, monkeypatch):
+ from repowise.server.mcp_server import _state, get_context
+
+ _write_source(tmp_path)
+ monkeypatch.setattr(_state, "_repo_path", str(tmp_path))
+ result = await get_context(["src/auth/service.py"], compact=False)
+ card = result["targets"]["src/auth/service.py"]
+ assert "skeleton" not in card
+ assert card["docs"]["symbols"]
+
+
+async def test_auto_skeleton_falls_back_to_card_when_source_missing(
+ setup_mcp, tmp_path, monkeypatch
+):
+ # Nothing on disk: the auto-upgrade must degrade to the symbol card, not
+ # to an error-only response (explicit include=["skeleton"] still errors).
+ from repowise.server.mcp_server import _state, get_context
+
+ monkeypatch.setattr(_state, "_repo_path", str(tmp_path))
+ result = await get_context(["src/auth/service.py"])
+ card = result["targets"]["src/auth/service.py"]
+ assert "skeleton" not in card
+ assert card["docs"]["symbols"]
diff --git a/tests/unit/server/mcp/test_exclude_spec.py b/tests/unit/server/mcp/test_exclude_spec.py
new file mode 100644
index 000000000..c498864c1
--- /dev/null
+++ b/tests/unit/server/mcp/test_exclude_spec.py
@@ -0,0 +1,27 @@
+"""_get_exclude_spec must union config excludes with the gitignore stack.
+
+Indexes built before the traverser honoured ``.git/info/exclude`` still
+contain rows for local-only scratch dirs; the query-time spec keeps those
+paths out of MCP tool responses without forcing a reindex.
+"""
+
+from __future__ import annotations
+
+from repowise.server.mcp_server._helpers import _get_exclude_spec, is_excluded
+
+
+def test_unions_gitignore_and_info_exclude(tmp_path):
+ (tmp_path / ".gitignore").write_text("dist/\n")
+ info = tmp_path / ".git" / "info"
+ info.mkdir(parents=True)
+ (info / "exclude").write_text("local-stash/\n")
+
+ spec = _get_exclude_spec(tmp_path)
+ assert spec is not None
+ assert is_excluded("local-stash/bench/probe.py", spec)
+ assert is_excluded("dist/bundle.js", spec)
+ assert not is_excluded("src/app.py", spec)
+
+
+def test_no_patterns_returns_none(tmp_path):
+ assert _get_exclude_spec(tmp_path) is None
diff --git a/tests/unit/server/mcp/test_search.py b/tests/unit/server/mcp/test_search.py
index 9fbc562fa..c4d665d79 100644
--- a/tests/unit/server/mcp/test_search.py
+++ b/tests/unit/server/mcp/test_search.py
@@ -31,6 +31,145 @@ async def test_search_codebase(setup_mcp):
assert len(result["results"]) >= 1
+def _mk_result(page_id, title, page_type, target_path, score):
+ from repowise.core.persistence.search import SearchResult
+
+ return SearchResult(
+ page_id=page_id,
+ title=title,
+ page_type=page_type,
+ target_path=target_path,
+ score=score,
+ snippet=title,
+ search_type="vector",
+ )
+
+
+class TestDecisionDownweight:
+ """Decision records must not crowd file pages out of the top ranks."""
+
+ def test_why_shaped_queries(self):
+ from repowise.server.mcp_server.tool_search import _is_why_shaped
+
+ assert _is_why_shaped("why is auth using JWT?")
+ assert _is_why_shaped("when did we switch to SQLite")
+ assert _is_why_shaped("who decided on LanceDB")
+ assert _is_why_shaped("what was the rationale for WAL mode")
+ assert _is_why_shaped("show me the decision about persistence")
+ assert not _is_why_shaped("where is the SQLite store for distilled output")
+ assert not _is_why_shaped("authentication flow")
+
+ def test_fetch_limit_always_overfetches(self):
+ from repowise.server.mcp_server.tool_search import _fetch_limit_for
+
+ # Down-weighting can only promote file pages that are inside the
+ # fetched window, so even unfiltered queries over-fetch.
+ assert _fetch_limit_for(5, None) == 15
+ assert _fetch_limit_for(5, "implementation") == 30
+
+ @pytest.mark.asyncio
+ async def test_file_page_outranks_downweighted_decision(self, setup_mcp):
+ import repowise.server.mcp_server as mcp_mod
+ from repowise.server.mcp_server import search_codebase
+
+ async def fake_search(query, limit=10):
+ return [
+ _mk_result("decision:d1", "Use repo-local SQLite", "decision_record", "", 0.57),
+ _mk_result(
+ "file_page:src/auth/service.py", "Auth Service", "file_page",
+ "src/auth/service.py", 0.42,
+ ),
+ ]
+
+ mcp_mod._vector_store.search = fake_search
+ result = await search_codebase("where is the SQLite store")
+ types = [r["page_type"] for r in result["results"]]
+ assert types[0] == "file_page"
+ # The decision page survives, just demoted below the file page.
+ assert "decision_record" in types
+
+ @pytest.mark.asyncio
+ async def test_why_query_keeps_decision_ranking(self, setup_mcp):
+ import repowise.server.mcp_server as mcp_mod
+ from repowise.server.mcp_server import search_codebase
+
+ async def fake_search(query, limit=10):
+ return [
+ _mk_result("decision:d1", "Use repo-local SQLite", "decision_record", "", 0.57),
+ _mk_result(
+ "file_page:src/auth/service.py", "Auth Service", "file_page",
+ "src/auth/service.py", 0.42,
+ ),
+ ]
+
+ mcp_mod._vector_store.search = fake_search
+ result = await search_codebase("why did we choose SQLite for persistence?")
+ assert result["results"][0]["page_type"] == "decision_record"
+
+ @pytest.mark.asyncio
+ async def test_kind_filter_survives_decision_flood(self, setup_mcp):
+ # 15 decision records ahead of one file page: with the old 3x
+ # over-fetch the window held only decisions and kind="implementation"
+ # returned []. The 6x window must surface the file page.
+ import repowise.server.mcp_server as mcp_mod
+ from repowise.server.mcp_server import search_codebase
+
+ flood = [
+ _mk_result(f"decision:d{i}", f"Decision {i}", "decision_record", "", 0.6 - i * 0.01)
+ for i in range(15)
+ ]
+ flood.append(
+ _mk_result(
+ "file_page:src/auth/service.py", "Auth Service", "file_page",
+ "src/auth/service.py", 0.35,
+ )
+ )
+
+ async def fake_search(query, limit=10):
+ return flood[:limit]
+
+ mcp_mod._vector_store.search = fake_search
+ result = await search_codebase("sqlite store", limit=5, kind="implementation")
+ paths = [r["target_path"] for r in result["results"]]
+ assert paths == ["src/auth/service.py"]
+
+ @pytest.mark.asyncio
+ async def test_federated_path_filters_kind_before_truncation(self, setup_mcp):
+ # _search_single_repo used to truncate to ``limit`` before the
+ # aggregate kind filter ran, so federated kind searches under-filled.
+ import types
+
+ import repowise.server.mcp_server as mcp_mod
+ from repowise.server.mcp_server.tool_search import _search_single_repo
+
+ flood = [
+ _mk_result(f"decision:d{i}", f"Decision {i}", "decision_record", "", 0.6 - i * 0.01)
+ for i in range(15)
+ ]
+ flood.append(
+ _mk_result(
+ "file_page:src/auth/service.py", "Auth Service", "file_page",
+ "src/auth/service.py", 0.35,
+ )
+ )
+
+ async def fake_search(query, limit=10):
+ return flood[:limit]
+
+ ctx = types.SimpleNamespace(
+ vector_store=types.SimpleNamespace(search=fake_search),
+ fts=mcp_mod._fts,
+ session_factory=mcp_mod._session_factory,
+ vector_store_ready=None,
+ path="/tmp/test-repo",
+ )
+ results, method = await _search_single_repo(
+ ctx, "sqlite store", limit=5, page_type=None, kind="implementation"
+ )
+ assert method == "embedding"
+ assert [r["target_path"] for r in results] == ["src/auth/service.py"]
+
+
class TestClassifyHitKind:
"""The ``kind`` filter's path heuristic."""