Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): deterministic coverage for the undocumented file tail (#817)

Raghav Chamadiya94659b042abf
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/command.py b/packages/cli/src/repowise/cli/commands/init_cmd/command.py
index 1ccde4186..2efd61756 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/command.py
@@ -94,6 +94,7 @@ def _run_generation_phase(
resolved_reasoning: str,
onboarding: bool,
tier1_top_n: int | None,
+ tier2_tail_enabled: bool,
harvest_decisions: bool,
wiki_style: str,
coverage_pct: float | None,
@@ -127,6 +128,7 @@ def _run_generation_phase(
reasoning=resolved_reasoning,
enable_onboarding=onboarding,
tier1_top_n=tier1_top_n,
+ tier2_tail_enabled=tier2_tail_enabled,
harvest_decisions=harvest_decisions,
wiki_style=wiki_style,
)
@@ -188,6 +190,17 @@ def _run_generation_phase(
)
return False, True
+ # Persist the tiering knobs so `repowise update` regenerates with the same
+ # coverage settings (save_config later round-trips and preserves these).
+ # save_config_partial skips None, so a no-cap tier1 stays unwritten.
+ from repowise.cli.helpers import save_config_partial
+
+ save_config_partial(
+ repo_path,
+ tier1_top_n=tier1_top_n,
+ tier2_tail_enabled=tier2_tail_enabled,
+ )
+
run_repo_generation(
repo_path=repo_path,
result=result,
@@ -634,6 +647,11 @@ def init_command(
# file page is a full-LLM tier-1 page (unchanged behaviour).
tier1_top_n: int | None = None
+ # Deterministic coverage tail (Phase G): document every remaining source
+ # file with a free, no-LLM page. On by default; only the advanced menu
+ # can turn it off.
+ tier2_tail_enabled: bool = True
+
# Output language picked in the advanced-mode generation section; None
# until chosen. Resolved below: flag > this > config.yaml > English.
language_choice: str | None = None
@@ -707,6 +725,7 @@ def init_command(
embedder_name = adv.get("embedder") or embedder_name
test_run = adv["test_run"]
tier1_top_n = adv.get("tier1_top_n")
+ tier2_tail_enabled = adv.get("tier2_tail_enabled", True)
onboarding = adv.get("onboarding", onboarding)
harvest_decisions = adv.get("harvest_decisions", harvest_decisions)
if adv.get("wiki_style"):
@@ -963,6 +982,7 @@ def init_command(
resolved_reasoning=resolved_reasoning,
onboarding=onboarding,
tier1_top_n=tier1_top_n,
+ tier2_tail_enabled=tier2_tail_enabled,
harvest_decisions=harvest_decisions,
wiki_style=wiki_style,
coverage_pct=coverage_pct,
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/generation.py b/packages/cli/src/repowise/cli/commands/init_cmd/generation.py
index 44757d30b..06cc28809 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/generation.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/generation.py
@@ -133,7 +133,9 @@ def select_coverage(
skip_infra=skip_infra,
kg_modules=kg_modules,
)
- chosen = interactive_coverage_select(console, options)
+ chosen = interactive_coverage_select(
+ console, options, deterministic_tail=gen_config.tier2_tail_enabled
+ )
chosen_pct = chosen.pct
plans = chosen.plans
est = chosen.estimate
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/reporting.py b/packages/cli/src/repowise/cli/commands/init_cmd/reporting.py
index 9d648fee9..ec9c9c543 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/reporting.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/reporting.py
@@ -200,9 +200,17 @@ def show_completion(
console.print()
_render_defect_accuracy(result)
else:
- total_tokens = sum(p.total_tokens for p in (result.generated_pages or []))
+ _pages = result.generated_pages or []
+ total_tokens = sum(p.total_tokens for p in _pages)
+ # Split AI-written from the zero-LLM deterministic coverage tail so broad
+ # coverage reads as thoroughness, not padding.
+ _det = sum(1 for p in _pages if getattr(p, "provider_name", "") == "template")
+ _ai = len(_pages) - _det
+ _pages_label = str(len(_pages))
+ if _det:
+ _pages_label = f"{len(_pages)} ({_ai} AI-written · {_det} deterministic)"
metrics = [
- ("Pages generated", str(len(result.generated_pages or []))),
+ ("Pages generated", _pages_label),
("Total tokens", f"{total_tokens:,}"),
("Provider", f"{provider.provider_name} / {provider.model_name}"),
("Elapsed", format_elapsed(elapsed)),
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
index 368f6db98..0f2d8780b 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
@@ -744,6 +744,10 @@ def run_update(
# on update yet — defaults to on to keep the onboarding collection
# fresh as the codebase evolves.
enable_onboarding_cfg = bool(cfg.get("enable_onboarding", True))
+ # Honor the tiering knobs chosen at init so update regenerates with the same
+ # coverage. Without reading these back, every update would silently drop the
+ # deterministic tail (and any tier-1 cap) to their defaults.
+ tail_dirs_cfg = cfg.get("tier2_tail_dirs")
config = GenerationConfig(
max_concurrency=concurrency,
language=language,
@@ -752,6 +756,10 @@ def run_update(
# Honor the wiki style chosen at init (or via `repowise restyle`) so pages
# regenerated for changed files match the rest of the wiki's voice.
wiki_style=cfg.get("wiki_style", "comprehensive"),
+ tier1_top_n=cfg.get("tier1_top_n"),
+ tier2_tail_enabled=bool(cfg.get("tier2_tail_enabled", True)),
+ tier2_tail_cap=cfg.get("tier2_tail_cap"),
+ tier2_tail_dirs=tuple(tail_dirs_cfg) if tail_dirs_cfg else None,
)
provider = resolve_provider(provider_name, model, repo_path=repo_path)
diff --git a/packages/cli/src/repowise/cli/coverage_select.py b/packages/cli/src/repowise/cli/coverage_select.py
index edcc63ae3..e6c519819 100644
--- a/packages/cli/src/repowise/cli/coverage_select.py
+++ b/packages/cli/src/repowise/cli/coverage_select.py
@@ -76,18 +76,28 @@ def render_coverage_table(options: list[CoverageOption]) -> Table:
def interactive_coverage_select(
console: Console,
options: list[CoverageOption],
+ *,
+ deterministic_tail: bool = False,
) -> CoverageOption:
"""Render the coverage table and prompt the user for a choice.
Returns the selected :class:`CoverageOption`. The default is the
one whose ``is_recommended`` flag is set, falling back to the
middle of the list when none is flagged.
+
+ ``deterministic_tail`` notes that files outside the chosen coverage still
+ get a free, no-LLM page (so the table's counts are not the whole story).
"""
if not options:
msg = "No coverage options provided"
raise ValueError(msg)
console.print(render_coverage_table(options))
+ if deterministic_tail:
+ console.print(
+ " [dim]Every file outside this coverage still gets a free, no-LLM "
+ "deterministic page, so search can find all of your code.[/dim]"
+ )
# Pick the default = recommended option's index, else the median.
default_idx = next(
diff --git a/packages/cli/src/repowise/cli/ui/mode_selection.py b/packages/cli/src/repowise/cli/ui/mode_selection.py
index e8c4d6719..50ee4b269 100644
--- a/packages/cli/src/repowise/cli/ui/mode_selection.py
+++ b/packages/cli/src/repowise/cli/ui/mode_selection.py
@@ -390,6 +390,20 @@ def _prompt_generation(
)
result["tier1_top_n"] = tier_val if tier_val > 0 else None
+ # Deterministic coverage tail: give every remaining (budget-dropped) source
+ # file a free, no-LLM page so the whole codebase is retrievable by search.
+ result["tier2_tail_enabled"] = True
+ if result["run_mode"] == "standard":
+ console.print()
+ console.print(
+ " [dim]Recommended: keep this on so search and get_answer can find "
+ "every source file, not just the documented slice (free, no tokens).[/dim]"
+ )
+ result["tier2_tail_enabled"] = click.confirm(
+ " Document remaining files deterministically (no-LLM coverage)?",
+ default=True,
+ )
+
# Documentation voice/density. An explicit --wiki-style wins; otherwise prompt
# here so the choice lands inside the section, before the summary panel.
result["wiki_style"] = wiki_style if wiki_style is not None else prompt_wiki_style(console)
@@ -446,6 +460,10 @@ def _build_summary_table(
)
if allow_fast and result.get("tier1_top_n"):
summary.add_row("Full-LLM page cap", str(result["tier1_top_n"]))
+ summary.add_row(
+ "Deterministic coverage",
+ "yes" if result.get("tier2_tail_enabled", True) else "no",
+ )
summary.add_row("Onboarding", "yes" if result.get("onboarding", True) else "no")
summary.add_row(
"Harvest decisions", "yes" if result.get("harvest_decisions", True) else "no"
diff --git a/packages/core/src/repowise/core/generation/models.py b/packages/core/src/repowise/core/generation/models.py
index 28e9bc625..8879cce69 100644
--- a/packages/core/src/repowise/core/generation/models.py
+++ b/packages/core/src/repowise/core/generation/models.py
@@ -163,6 +163,21 @@ class GenerationConfig:
# ``None`` (default) preserves the prior behaviour exactly: every
# selected file page is a full tier-1 LLM page.
tier1_top_n: int | None = None
+ # ---- Deterministic coverage tail (Phase G) ------------------------
+ # After the budget picks its LLM (tier-1/2) file pages, every REMAINING
+ # parsed source file gets a cheap, zero-LLM "deterministic" page (the same
+ # template renderer as tier-2), so the whole codebase is retrievable by
+ # concept search instead of only the ~20% the budget covers. Proven in
+ # dogfood to lift retrieval recall (raw-vector recall@5 0.47 -> 0.67) with
+ # no regressions once the tail is importance-floored (test files and pure
+ # __init__.py re-exports are always excluded — they only dilute retrieval).
+ # On by default; free (no tokens), costs only index size + embeddings.
+ tier2_tail_enabled: bool = True
+ # Optional cap on how many tail pages to emit (highest-signal first by
+ # score). None = every floored candidate. Use to bound index size.
+ tier2_tail_cap: int | None = None
+ # Optional directory allow-list (repo-relative prefixes). None = all dirs.
+ tier2_tail_dirs: tuple[str, ...] | None = None
def __post_init__(self) -> None:
if self.embed_concurrency is None:
diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.py
index 9762ed271..ba2494ae5 100644
--- a/packages/core/src/repowise/core/generation/page_generator/core.py
+++ b/packages/core/src/repowise/core/generation/page_generator/core.py
@@ -348,14 +348,23 @@ class PageGenerator(PerTypeGenerationMixin):
self,
parsed: ParsedFile,
ctx: FilePageContext,
+ *,
+ tail: bool = False,
) -> GeneratedPage:
"""Render a deterministic (no-LLM) tier-2 file page.
- Used for the long tail of selected files on large repos. The page is
- built straight from the assembled context via a Jinja template, marked
- as template-generated, and carries zero token cost. It is embedded for
- search by the level runner like any other page. No provider call and
- no hallucination check (the content is factual by construction).
+ Used for the long tail of selected files on large repos, and (with
+ ``tail=True``) for the Phase G coverage tail: code files the budget
+ dropped entirely, so the whole codebase is retrievable by concept
+ search. The page is built straight from the assembled context via a
+ Jinja template, marked as template-generated, and carries zero token
+ cost. It is embedded for search by the level runner like any other
+ page. No provider call and no hallucination check (the content is
+ factual by construction).
+
+ ``tail=True`` stamps ``doc_tier=3`` (vs 2 for the in-budget tail) so
+ serving/ranking and the UI can tell budget-tail coverage pages apart
+ and rank them below LLM pages; ``metadata["deterministic"]`` marks both.
"""
content = self._render("file_page_tier2.j2", style_prefix=False, ctx=ctx)
now = _now_iso()
@@ -380,7 +389,8 @@ class PageGenerator(PerTypeGenerationMixin):
created_at=now,
updated_at=now,
)
- page.metadata["doc_tier"] = 2
+ page.metadata["doc_tier"] = 3 if tail else 2
+ page.metadata["deterministic"] = True
_attach_file_provenance(page, ctx)
return page
diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.py
index 71743a663..ff8989462 100644
--- a/packages/core/src/repowise/core/generation/page_generator/levels.py
+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py
@@ -178,9 +178,7 @@ async def _prefetch_rag_context(
return False
for (p, ctx), results in zip(targets, all_results, strict=False):
self_id = f"file_page:{p.file_info.path}"
- ctx.rag_context = [
- f"[{r.page_id}]\n{r.snippet}" for r in results if r.page_id != self_id
- ]
+ ctx.rag_context = [f"[{r.page_id}]\n{r.snippet}" for r in results if r.page_id != self_id]
return True
@@ -203,7 +201,9 @@ async def build_level2_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
items: list[tuple[Any, FilePageContext]] = []
for p in run.code_files:
- kg_file_ctx = run.kg_ctx.get_file_context(p.file_info.path) if run.kg_ctx.available else None
+ kg_file_ctx = (
+ run.kg_ctx.get_file_context(p.file_info.path) if run.kg_ctx.available else None
+ )
ctx: FilePageContext = gen._assembler.assemble_file_page(
p,
run.graph,
@@ -234,16 +234,22 @@ async def build_level2_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
)
coros: list[tuple[str, Any]] = []
+ tail_paths = getattr(run, "tail_paths", set())
for p, ctx in items:
path = p.file_info.path
pid = compute_page_id("file_page", path)
- if path in run.sel_file_paths and pid not in run.completed_ids:
+ if pid in run.completed_ids:
+ continue
+ if path in run.sel_file_paths:
if path in run.tier1_paths:
coros.append(
(pid, gen._generate_file_page_from_ctx(p, ctx, rag_prefetched=rag_prefetched))
)
else:
coros.append((pid, gen._generate_file_page_tier2(p, ctx)))
+ elif path in tail_paths:
+ # Phase G coverage tail: budget-dropped code file -> zero-LLM page.
+ coros.append((pid, gen._generate_file_page_tier2(p, ctx, tail=True)))
return coros
@@ -252,9 +258,7 @@ def build_level3_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
gen = run.gen
coros: list[tuple[str, Any]] = []
for scc_id, scc_files in run.sel_scc_groups:
- fc_list = [
- run.file_page_contexts[f] for f in scc_files if f in run.file_page_contexts
- ]
+ fc_list = [run.file_page_contexts[f] for f in scc_files if f in run.file_page_contexts]
pid = compute_page_id("scc_page", scc_id)
if pid not in run.completed_ids:
coros.append((pid, gen.generate_scc_page(scc_id, scc_files, fc_list)))
@@ -266,11 +270,7 @@ def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
gen = run.gen
coros: list[tuple[str, Any]] = []
for mg in run.sel_module_groups:
- fcs = [
- run.file_page_contexts[fp]
- for fp in mg.file_paths
- if fp in run.file_page_contexts
- ]
+ fcs = [run.file_page_contexts[fp] for fp in mg.file_paths if fp in run.file_page_contexts]
if not fcs:
continue
page_id = compute_page_id("module_page", mg.key)
@@ -413,9 +413,7 @@ def build_level7_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
"""Level 7 (infra_page), allow-set filtered."""
gen = run.gen
infra_files = [
- p
- for p in run.parsed_files
- if _is_infra_file(p) and p.file_info.path in run.sel_infra_paths
+ p for p in run.parsed_files if _is_infra_file(p) and p.file_info.path in run.sel_infra_paths
]
return [
(
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index 98d1bd019..f53e8cd21 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -177,11 +177,7 @@ class _GenerationRun:
)
async def _seed_resume(self) -> None:
- if (
- self.job_system is not None
- and self.resume
- and self.vector_store is not None
- ):
+ if self.job_system is not None and self.resume and self.vector_store is not None:
self.completed_ids = await self.vector_store.list_page_ids()
if self.completed_ids:
log.info(
@@ -260,6 +256,11 @@ class _GenerationRun:
kg_file_scores=kg_scores or None,
)
+ # Deterministic coverage tail (Phase G): code files the budget dropped,
+ # rendered by the same zero-LLM template path as tier-2. Disjoint from
+ # sel_file_paths by construction (select_pages excludes the selected set).
+ self.tail_paths = set(getattr(selection, "deterministic_tail_paths", []) or [])
+
# Sort code_files for stable level-2 ordering: selected files first
# (so dep summaries land in the store earliest), then by PageRank desc.
self.code_files = sorted(
@@ -302,7 +303,8 @@ class _GenerationRun:
layer_page_count = 0
if self.kg_ctx.available:
layer_page_count = sum(
- 1 for l in self.kg_ctx.get_layers()
+ 1
+ for l in self.kg_ctx.get_layers()
if len([n for n in l.get("nodeIds", []) if n.startswith("file:")]) >= 3
)
estimated_total = (
@@ -315,6 +317,8 @@ class _GenerationRun:
+ int(self.selection.emit_repo_overview)
+ int(self.selection.emit_arch_diagram)
+ counts["infra_page"]
+ # Deterministic coverage tail also produces (zero-LLM) pages.
+ + len(getattr(self.selection, "deterministic_tail_paths", []))
)
remaining_total = max(0, estimated_total - len(self.completed_ids))
if self.on_total_known is not None:
@@ -376,9 +380,9 @@ class _GenerationRun:
file_layers: dict[str, str] = {}
for path in self.sel_file_paths:
kg_fc = self.kg_ctx.get_file_context(path) if self.kg_ctx.available else None
- file_layers[path] = (kg_fc.layer_name if kg_fc and kg_fc.layer_name else "") or infer_layer(
- path, lang_by_path.get(path)
- )
+ file_layers[path] = (
+ kg_fc.layer_name if kg_fc and kg_fc.layer_name else ""
+ ) or infer_layer(path, lang_by_path.get(path))
self.layer_order = compute_layer_order(file_layers, import_edges)
# ------------------------------------------------------------------
@@ -547,7 +551,9 @@ class _GenerationRun:
try:
from ..kg_enrichment import enrich_tour_with_wiki_links
- rp = Path(self.repo_path) if not isinstance(self.repo_path, Path) else self.repo_path
+ rp = (
+ Path(self.repo_path) if not isinstance(self.repo_path, Path) else self.repo_path
+ )
kg_path = rp / ".repowise" / "knowledge-graph.json"
if kg_path.exists():
enrich_tour_with_wiki_links(kg_path, all_pages)
diff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.py
index 95d59f5ea..5fe7118eb 100644
--- a/packages/core/src/repowise/core/generation/selection/selector.py
+++ b/packages/core/src/repowise/core/generation/selection/selector.py
@@ -69,7 +69,9 @@ class Selection:
"""Allow-set returned by :func:`select_pages`."""
file_page_paths: list[str] = field(default_factory=list)
- symbol_spotlights: list[tuple[str, str]] = field(default_factory=list) # (file_path, symbol_name)
+ symbol_spotlights: list[tuple[str, str]] = field(
+ default_factory=list
+ ) # (file_path, symbol_name)
module_groups: list[ModuleGroup] = field(default_factory=list)
api_contract_paths: list[str] = field(default_factory=list)
infra_paths: list[str] = field(default_factory=list)
@@ -77,9 +79,18 @@ class Selection:
emit_repo_overview: bool = True
emit_arch_diagram: bool = True
allocation: BucketAllocation | None = None
+ # Zero-LLM deterministic pages for the code files the budget did NOT pick
+ # (Phase G coverage tail). Kept separate from ``file_page_paths`` so cost
+ # estimation stays honest (these are free) and the CLI can report the split.
+ deterministic_tail_paths: list[str] = field(default_factory=list)
def counts(self) -> dict[str, int]:
- """Per-page-type counts (for cost estimation and the init UI)."""
+ """Per-page-type counts of BUDGETED (LLM-costed) pages.
+
+ Deliberately excludes ``deterministic_tail_paths`` — those are free
+ zero-LLM pages, reported separately (``len(deterministic_tail_paths)``)
+ so cost estimation and the budget contract are not inflated by them.
+ """
return {
"api_contract": len(self.api_contract_paths),
"symbol_spotlight": len(self.symbol_spotlights),
@@ -142,11 +153,46 @@ def _is_infra_file(parsed: Any) -> bool:
def _is_code_file(parsed: Any) -> bool:
fi = parsed.file_info
- return (
- not fi.is_api_contract
- and not _is_infra_file(parsed)
- and fi.language in _CODE_LANGUAGES
- )
+ return not fi.is_api_contract and not _is_infra_file(parsed) and fi.language in _CODE_LANGUAGES
+
+
+def _passes_tail_floor(path: str, tail_dirs: tuple[str, ...] | None) -> bool:
+ """Importance floor for the deterministic coverage tail (Phase G).
+
+ Two exclusions are ALWAYS applied because they were proven to only dilute
+ retrieval (test-file pages pushed real answers below rank 5 in dogfood):
+ test files and pure ``__init__.py`` re-export files. When ``tail_dirs`` is
+ set, the path must also live under one of those repo-relative prefixes.
+ """
+ norm = path.replace("\\", "/")
+ if norm.startswith("tests/") or "/tests/" in norm:
+ return False
+ if norm.rsplit("/", 1)[-1] == "__init__.py":
+ return False
+ if tail_dirs:
+ return any(norm == d.rstrip("/") or norm.startswith(d.rstrip("/") + "/") for d in tail_dirs)
+ return True
+
+
+def _select_deterministic_tail(
+ files: list[tuple[float, str]],
+ selected_files: list[str],
+ cfg: Any,
+) -> list[str]:
+ """Every code file the budget dropped, importance-floored and capped.
+
+ ``files`` is score-descending, so a cap keeps the highest-signal tail.
+ Returns [] when the tail is disabled, reproducing the prior behaviour.
+ """
+ if not getattr(cfg, "tier2_tail_enabled", True):
+ return []
+ selected = set(selected_files)
+ tail_dirs = getattr(cfg, "tier2_tail_dirs", None)
+ tail = [p for _, p in files if p not in selected and _passes_tail_floor(p, tail_dirs)]
+ cap = getattr(cfg, "tier2_tail_cap", None)
+ if cap is not None and cap >= 0:
+ tail = tail[:cap]
+ return tail
# ---------------------------------------------------------------------------
@@ -219,9 +265,7 @@ def _build_curated_module_groups(
if not inputs.kg_modules:
return None
- code_by_path = {
- p.file_info.path: p for p in inputs.parsed_files if _is_code_file(p)
- }
+ code_by_path = {p.file_info.path: p for p in inputs.parsed_files if _is_code_file(p)}
scored: list[tuple[float, ModuleGroup]] = []
seen_keys: set[str] = set()
for module in inputs.kg_modules:
@@ -233,8 +277,7 @@ def _build_curated_module_groups(
member_paths = sorted(
path
for nid in module.get("nodeIds", [])
- if isinstance(nid, str)
- and (path := nid.removeprefix("file:")) in code_by_path
+ if isinstance(nid, str) and (path := nid.removeprefix("file:")) in code_by_path
)
if len(member_paths) < min_size:
continue
@@ -246,9 +289,7 @@ def _build_curated_module_groups(
continue
seen_keys.add(key)
name = module.get("name") or key
- language = module.get("language") or code_by_path[
- member_paths[0]
- ].file_info.language
+ language = module.get("language") or code_by_path[member_paths[0]].file_info.language
score = sum(inputs.pagerank.get(p, 0.0) for p in member_paths)
scored.append(
(
@@ -550,8 +591,13 @@ def select_pages(inputs: SelectionInputs) -> Selection:
]
selected_files = _ensure_landmarks(selected_files, landmarks)
+ # Deterministic coverage tail: every code file the budget dropped gets a
+ # cheap zero-LLM page so the whole codebase is retrievable (Phase G).
+ deterministic_tail = _select_deterministic_tail(files, selected_files, cfg)
+
sel = Selection(
file_page_paths=selected_files,
+ deterministic_tail_paths=deterministic_tail,
symbol_spotlights=[t for _, t in symbols[: allocation.symbol_spotlight]],
module_groups=[m for _, m in modules[: allocation.module_page]],
api_contract_paths=[p for _, p in apis[: allocation.api_contract]],
diff --git a/tests/integration/test_generation_pipeline.py b/tests/integration/test_generation_pipeline.py
index 45d77e29a..cdfb3aaa0 100644
--- a/tests/integration/test_generation_pipeline.py
+++ b/tests/integration/test_generation_pipeline.py
@@ -319,8 +319,28 @@ class TestGenerationPipeline:
assert page.model_name
def test_all_pages_provider_is_mock(self, pipeline_result):
+ # LLM pages use the mock provider; the deterministic coverage tail
+ # (files outside the budget) is rendered from a template with no
+ # provider call, so those carry provider_name "template".
for page in pipeline_result["pages"]:
- assert page.provider_name == "mock"
+ assert page.provider_name in ("mock", "template")
+
+ def test_deterministic_tail_pages_emitted(self, pipeline_result):
+ """Files outside the LLM budget get zero-LLM deterministic file pages.
+
+ With the default config (tier1_top_n=None) the only template pages are
+ the Phase G coverage tail, tagged doc_tier=3.
+ """
+ tail = [
+ p
+ for p in pipeline_result["pages"]
+ if p.provider_name == "template" and p.metadata.get("doc_tier") == 3
+ ]
+ assert tail, "expected deterministic coverage-tail pages"
+ for p in tail:
+ assert p.page_type == "file_page"
+ assert p.input_tokens == 0 and p.output_tokens == 0
+ assert p.metadata.get("deterministic") is True
def test_generates_repo_overview(self, pipeline_result):
types = [p.page_type for p in pipeline_result["pages"]]
@@ -403,8 +423,14 @@ class TestGenerationPipeline:
assert jobs[0].completed_pages >= 0
def test_generated_page_tokens_nonzero(self, pipeline_result):
- """Every generated page should report nonzero token usage."""
- for page in pipeline_result["pages"]:
+ """Every LLM page should report nonzero token usage.
+
+ Deterministic tail pages (provider "template") are zero-LLM by design,
+ so they are excluded from this token assertion.
+ """
+ llm_pages = [p for p in pipeline_result["pages"] if p.provider_name != "template"]
+ assert llm_pages, "expected at least one LLM page"
+ for page in llm_pages:
assert page.input_tokens > 0 or page.output_tokens > 0
def test_generated_page_source_hash_is_hex(self, pipeline_result):
diff --git a/tests/unit/cli/test_ui_package.py b/tests/unit/cli/test_ui_package.py
index 0895cde35..af62b6a81 100644
--- a/tests/unit/cli/test_ui_package.py
+++ b/tests/unit/cli/test_ui_package.py
@@ -117,6 +117,7 @@ def test_advanced_config_default_keys_no_fast(monkeypatch: pytest.MonkeyPatch) -
"onboarding": True,
"harvest_decisions": True,
"tier1_top_n": None,
+ "tier2_tail_enabled": True,
"wiki_style": DEFAULT_STYLE,
"language": "en",
}
diff --git a/tests/unit/generation/test_selection_deterministic_tail.py b/tests/unit/generation/test_selection_deterministic_tail.py
new file mode 100644
index 000000000..a7816a135
--- /dev/null
+++ b/tests/unit/generation/test_selection_deterministic_tail.py
@@ -0,0 +1,99 @@
+"""Deterministic coverage-tail tests for the selection layer (Phase G).
+
+Contract: after the LLM budget picks its file pages, every REMAINING code file
+gets a zero-LLM deterministic page (``Selection.deterministic_tail_paths``),
+importance-floored (test files and pure ``__init__.py`` always excluded) and
+optionally capped / dir-restricted. Disabling the tail reproduces prior output.
+"""
+
+from __future__ import annotations
+
+from repowise.core.generation.models import GenerationConfig
+from repowise.core.generation.selection import SelectionInputs, select_pages
+
+from .test_selection_budget import (
+ FakeFileInfo,
+ FakeParsedFile,
+ FakeSymbol,
+ _build_synthetic_repo,
+)
+
+
+def _inputs(parsed, pagerank, betweenness, community, cfg) -> SelectionInputs:
+ return SelectionInputs(
+ parsed_files=parsed,
+ pagerank=pagerank,
+ betweenness=betweenness,
+ community=community,
+ community_info=None,
+ sccs=[],
+ git_meta_map=None,
+ config=cfg,
+ )
+
+
+def test_tail_covers_every_dropped_code_file():
+ parsed, pr, bet, comm = _build_synthetic_repo(200)
+ cfg = GenerationConfig(coverage_pct=0.10) # ~20 file pages, ~180 dropped
+ sel = select_pages(_inputs(parsed, pr, bet, comm, cfg))
+
+ selected = set(sel.file_page_paths)
+ tail = set(sel.deterministic_tail_paths)
+ # No overlap between the budgeted set and the tail.
+ assert not (selected & tail)
+ # Every code file is covered by exactly one of the two (no test/__init__
+ # files in this synthetic repo, so the union is the full set).
+ all_code = {p.file_info.path for p in parsed}
+ assert selected | tail == all_code
+
+
+def test_tail_floor_excludes_tests_and_init():
+ parsed, pr, bet, comm = _build_synthetic_repo(30)
+ # Inject a test file and an __init__.py that would otherwise land in the tail.
+ for path in ("tests/test_thing.py", "pkg0/__init__.py", "src/tests/helper.py"):
+ fi = FakeFileInfo(path=path)
+ parsed.append(FakeParsedFile(file_info=fi, symbols=[FakeSymbol(name="x")]))
+ pr[path] = 0.001
+ bet[path] = 0.0
+ comm[path] = 0
+ cfg = GenerationConfig(coverage_pct=0.10)
+ sel = select_pages(_inputs(parsed, pr, bet, comm, cfg))
+
+ tail = set(sel.deterministic_tail_paths)
+ assert "tests/test_thing.py" not in tail
+ assert "src/tests/helper.py" not in tail
+ assert "pkg0/__init__.py" not in tail
+
+
+def test_tail_disabled_yields_empty():
+ parsed, pr, bet, comm = _build_synthetic_repo(100)
+ cfg = GenerationConfig(coverage_pct=0.10, tier2_tail_enabled=False)
+ sel = select_pages(_inputs(parsed, pr, bet, comm, cfg))
+ assert sel.deterministic_tail_paths == []
+
+
+def test_tail_respects_cap_highest_signal_first():
+ parsed, pr, bet, comm = _build_synthetic_repo(200)
+ cfg = GenerationConfig(coverage_pct=0.10, tier2_tail_cap=25)
+ sel = select_pages(_inputs(parsed, pr, bet, comm, cfg))
+ assert len(sel.deterministic_tail_paths) == 25
+ # Capped tail should be the highest-PageRank dropped files (score-ordered).
+ tail_prs = [pr[p] for p in sel.deterministic_tail_paths]
+ assert tail_prs == sorted(tail_prs, reverse=True)
+
+
+def test_tail_dir_restrict():
+ parsed, pr, bet, comm = _build_synthetic_repo(100)
+ cfg = GenerationConfig(coverage_pct=0.05, tier2_tail_dirs=("pkg1/",))
+ sel = select_pages(_inputs(parsed, pr, bet, comm, cfg))
+ assert sel.deterministic_tail_paths # non-empty
+ assert all(p.startswith("pkg1/") for p in sel.deterministic_tail_paths)
+
+
+def test_counts_excludes_tail():
+ """counts() stays budget-only so cost estimation isn't inflated."""
+ parsed, pr, bet, comm = _build_synthetic_repo(200)
+ cfg = GenerationConfig(coverage_pct=0.10)
+ sel = select_pages(_inputs(parsed, pr, bet, comm, cfg))
+ assert "deterministic_tail" not in sel.counts()
+ assert sel.deterministic_tail_paths # but the tail is populated