Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(wiki): drop dead code from the module prompt, keep file vocabulary out of the page, honest footer and model stamp (#2681)
Swati Ahuja65fcde37e5e6
diff --git a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.pyindex 8e2ea4fdb..a622e9f65 100644--- a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py+++ b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py@@ -881,6 +881,7 @@ def _run_repo_checks(target_path=page.target_path or "",summary=page.summary or "",content=page.content or "",+ page_metadata=page.metadata_json,)if item is None:# Below the information floor, so its absencediff --git a/packages/cli/src/repowise/cli/commands/reindex_cmd.py b/packages/cli/src/repowise/cli/commands/reindex_cmd.pyindex adb5e2232..70b77e8db 100644--- a/packages/cli/src/repowise/cli/commands/reindex_cmd.py+++ b/packages/cli/src/repowise/cli/commands/reindex_cmd.py@@ -214,6 +214,7 @@ async def _reindex(repo_path, embedder_name: str, batch_size: int) -> None:target_path=page.target_path or "",summary=page.summary or "",content=page.content or "",+ page_metadata=page.metadata_json,)if item is None:# Below the information floor. Not a failure — the page isdiff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.pyindex 90ada550f..3cb54dff2 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py@@ -1659,6 +1659,7 @@ def run_update(target_path=page.target_path,summary=page.summary,content=page.content,+ page_metadata=page.metadata,))is not Nonediff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.pyindex 82d30b14b..233a1752d 100644--- a/packages/core/src/repowise/core/generation/page_generator/core.py+++ b/packages/core/src/repowise/core/generation/page_generator/core.py@@ -23,6 +23,7 @@ import jinja2import structlogfrom repowise.core.ingestion.models import ParsedFile, RepoStructure+from repowise.core.persistence.vector_store import FILE_VOCABULARY_KEYfrom repowise.core.providers.llm.base import BaseProvider, CacheHint, GeneratedResponsefrom ..context.evidence import (@@ -336,6 +337,9 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):# file template does not emit; keep this type's original extraction.page.summary = _extract_summary(page.content)_attach_file_provenance(page, ctx)+ # Embedded for search, not rendered (see file_page.j2).+ if ctx.file_vocabulary:+ page.metadata[FILE_VOCABULARY_KEY] = ctx.file_vocabularyreturn page# ------------------------------------------------------------------diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.pyindex 77f09289c..4ca635e10 100644--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py@@ -1051,6 +1051,7 @@ def _embed_item(page: GeneratedPage) -> tuple[str, str, dict] | None:target_path=page.target_path,summary=page.summary,content=page.content,+ page_metadata=page.metadata,)diff --git a/packages/core/src/repowise/core/generation/page_generator/structural.py b/packages/core/src/repowise/core/generation/page_generator/structural.pyindex dc97ea65b..16ec5dfdf 100644--- a/packages/core/src/repowise/core/generation/page_generator/structural.py+++ b/packages/core/src/repowise/core/generation/page_generator/structural.py@@ -692,7 +692,7 @@ def stale_spotlight_paths(class StructuralRenderMixin:"""Template-only renderers, mixed into PageGenerator.- Requires the host to provide ``_render``, ``_provider`` and ``_config``.+ Requires the host to provide ``_render`` and ``_config``."""def _render_page(@@ -707,7 +707,8 @@ class StructuralRenderMixin:"""Render one template page and wrap it as a GeneratedPage.The mirror of ``_build_generated_page`` for the no-model path: same- fields, zero tokens, ``provider_name="template"``.+ fields, zero tokens, ``provider_name="template"``, and no model name:+ no model wrote this page, so it must not be credited to one.Confidence is a constant here, not a parameter. It used to beoverridable so a stub could claim less than a sole renderer's page,@@ -727,7 +728,7 @@ class StructuralRenderMixin:content=content,summary=_extract_summary(content, skip_metadata=True),source_hash=compute_source_hash(content),- model_name=self._provider.model_name,+ model_name="",provider_name="template",input_tokens=0,output_tokens=0,diff --git a/packages/core/src/repowise/core/generation/structural_labels.py b/packages/core/src/repowise/core/generation/structural_labels.pyindex b27165978..fe73cc9aa 100644--- a/packages/core/src/repowise/core/generation/structural_labels.py+++ b/packages/core/src/repowise/core/generation/structural_labels.py@@ -33,11 +33,7 @@ ENGLISH_LABELS: dict[str, str] = {# -- shared ------------------------------------------------------------"overview": "Overview","source": "Source",- "footer": (- "*Built from the code itself: parsed symbols, the import graph, git history and\n"- "the knowledge graph. Every statement here is checked against the source rather\n"- "than written about it.*"- ),+ "footer": "*Generated from parsed code, the import graph and git history.*","and_more": "and {count} more.","file": "File","file_singular": "file",@@ -99,7 +95,6 @@ ENGLISH_LABELS: dict[str, str] = {"changes_with_entry": "{count} shared {commit_word}","last_together": "last together on {date}","decisions_heading": "Decisions touching this file",- "in_the_code": "In the code","question_exports": "What does `{path}` export?","question_where_defined": "Where is `{symbol}` defined?","question_what_imports": "What imports `{path}`?",@@ -188,11 +183,7 @@ LOCALIZED_LABELS: dict[str, dict[str, str]] = {"de": {"overview": "Überblick","source": "Quelltext",- "footer": (- "*Aus dem Code selbst erstellt: geparste Symbole, der Importgraph, die "- "Git-Historie\nund der Wissensgraph. Jede Aussage hier wird gegen den "- "Quelltext geprüft, statt\nnur darüber geschrieben zu werden.*"- ),+ "footer": "*Erstellt aus geparstem Code, dem Importgraphen und der Git-Historie.*","and_more": "und {count} weitere.","file": "Datei","file_singular": "Datei",@@ -247,7 +238,6 @@ LOCALIZED_LABELS: dict[str, dict[str, str]] = {"changes_with_entry": "{count} gemeinsame {commit_word}","last_together": "zuletzt gemeinsam am {date}","decisions_heading": "Entscheidungen zu dieser Datei",- "in_the_code": "Im Code","question_exports": "Was exportiert `{path}`?","question_where_defined": "Wo ist `{symbol}` definiert?","question_what_imports": "Was importiert `{path}`?",diff --git a/packages/core/src/repowise/core/generation/templates/file_page.j2 b/packages/core/src/repowise/core/generation/templates/file_page.j2index 4466e2673..b1b3e3d5a 100644--- a/packages/core/src/repowise/core/generation/templates/file_page.j2+++ b/packages/core/src/repowise/core/generation/templates/file_page.j2@@ -206,29 +206,10 @@ _{{ labels.and_more.format(count=ctx.dependents | length - 25) }}_- {{ q }}{%- endfor %}{%- endif %}-{#- The words the file itself uses.-- Everything above is a description of the file. This is the file's own- vocabulary: field names, string literals, help text, comment prose. The- page could describe a file accurately and still be unfindable by any- question about what it does, because a question is asked in the words of- the source and none of those words were on the page.-- Per-file, and that is the whole point. Repo-level vocabulary rendered here- would be byte-identical across every file page in the corpus, which is the- failure this template's own header warns about: it would match every query- and distinguish none of them.-- Last on the page, below the questions block, for the same reason that- block is low: ``_extract_summary`` reads back from the top, and a bag of- words is not a summary. Conditional, like every section below the- Overview, so a file that yields no vocabulary renders no heading.--#}-{%- if ctx.file_vocabulary %}--## {{ labels.in_the_code }}--{{ ctx.file_vocabulary }}-{%- endif %}+{#- The file's own vocabulary (``ctx.file_vocabulary``) is not rendered.+ ``_render_file_page`` stores it in page metadata and ``embed_item`` adds+ it to the embedded text: it is a bag of words, which serves retrieval and+ reads to a person as noise.+#}{% include "_footer.j2" %}diff --git a/packages/core/src/repowise/core/generation/templates/module_page.j2 b/packages/core/src/repowise/core/generation/templates/module_page.j2index e0e58bfdd..754c1a47c 100644--- a/packages/core/src/repowise/core/generation/templates/module_page.j2+++ b/packages/core/src/repowise/core/generation/templates/module_page.j2@@ -109,13 +109,6 @@ These modules change together with this one in history and nothing in the depend{% endfor %}{% endif %}-{% if ctx.dead_code_findings %}-### Potentially unused code-{% for finding in ctx.dead_code_findings[:10] %}-- `{{ finding.symbol_name or '?' }}` — {{ finding.reason }} ({{ (finding.confidence * 100) | int }}%{% if finding.safe_to_delete %}, cleanup-ready{% endif %})-{% endfor %}-{% endif %}-{% if module_git_summary is defined and module_git_summary %}### Recent activityMost active file: `{{ module_git_summary.most_active_file }}` ({{ module_git_summary.most_active_commits_90d }} commits in 90 days).diff --git a/packages/core/src/repowise/core/index_scope.py b/packages/core/src/repowise/core/index_scope.pyindex 5ce09fa14..4b300529c 100644--- a/packages/core/src/repowise/core/index_scope.py+++ b/packages/core/src/repowise/core/index_scope.py@@ -114,7 +114,7 @@ def resolve_index_scope(git_tier = _choice(state.get("git_tier", scope.get("git_tier")), {"essential", "full"})if isinstance(stored, Mapping):provenance = _choice(- scope.get("content_provenance", provenance), {"none", "template", "model"}+ scope.get("content_provenance", provenance), {"none", "template", "model", "mixed"})unavailable = analysis.get("unavailable", [])diff --git a/packages/core/src/repowise/core/persistence/vector_store/__init__.py b/packages/core/src/repowise/core/persistence/vector_store/__init__.pyindex f83f059d6..10070ea5b 100644--- a/packages/core/src/repowise/core/persistence/vector_store/__init__.py+++ b/packages/core/src/repowise/core/persistence/vector_store/__init__.py@@ -27,6 +27,7 @@ to avoid one embedding round-trip per page; the single-itemfrom __future__ import annotationsfrom ._base import (+ FILE_VOCABULARY_KEY,BatchChunkFailure,BatchEmbeddingError,VectorStore,@@ -39,6 +40,7 @@ from .lancedb_store import LanceDBVectorStorefrom .pgvector_store import PgVectorStore__all__ = [+ "FILE_VOCABULARY_KEY","BatchChunkFailure","BatchEmbeddingError","InMemoryVectorStore",diff --git a/packages/core/src/repowise/core/persistence/vector_store/_base.py b/packages/core/src/repowise/core/persistence/vector_store/_base.pyindex acc6b31fb..bbce123c8 100644--- a/packages/core/src/repowise/core/persistence/vector_store/_base.py+++ b/packages/core/src/repowise/core/persistence/vector_store/_base.py@@ -8,10 +8,11 @@ re-exported from the package ``__init__`` so the historical import pathfrom __future__ import annotations+import jsonimport loggingimport mathfrom abc import ABC, abstractmethod-from collections.abc import Iterator+from collections.abc import Iterator, Mappingfrom dataclasses import dataclassfrom typing import Literal@@ -21,6 +22,7 @@ from ..search import SearchResult__all__ = ["EMBED_BATCH_MAX_ITEMS","EMBED_TEXT_MAX_CHARS",+ "FILE_VOCABULARY_KEY","STORED_SNIPPET_CHARS","BatchChunkFailure","BatchEmbeddingError",@@ -95,6 +97,11 @@ EMBED_TEXT_MAX_CHARS = 16_000# and on the paths that passed no content at all, an empty string.STORED_SNIPPET_CHARS = 2_000+# Page-metadata key for a file page's own vocabulary (field names, string+# literals, comment prose). It is embedded with the page but not rendered on+# it: a bag of words helps retrieval and reads to a person as noise.+FILE_VOCABULARY_KEY = "file_vocabulary"+def cap_embed_text(page_id: str, text: str) -> str:"""Return text bounded for one embedding input and report any loss.@@ -126,6 +133,7 @@ def embed_item(target_path: str,summary: str,content: str,+ page_metadata: Mapping[str, object] | str | None = None,) -> tuple[str, str, dict] | None:"""Build the one ``(page_id, text, metadata)`` item every writer embeds.@@ -164,6 +172,11 @@ def embed_item(resolves as a link target, and a reader who arrives at it still learns thefile exists. It is only kept out of the index, where its cost is paid byother pages. The floor is 0 by default, which admits everything.++ ``page_metadata`` is the page's metadata, as a dict (a generated page) or+ the stored JSON string (a ``wiki_pages`` row). Only+ :data:`FILE_VOCABULARY_KEY` is read from it, and appended after the+ content; the floor still measures ``content`` alone."""if not title.strip():raise ValueError(@@ -174,7 +187,7 @@ def embed_item(if not meets_information_floor(content):count_page_denied_a_vector()return None- parts = [p for p in (title, target_path, summary, content) if p]+ parts = [p for p in (title, target_path, summary, content, _vocabulary(page_metadata)) if p]return (page_id,"\n".join(parts),@@ -190,6 +203,18 @@ def embed_item()+def _vocabulary(page_metadata: Mapping[str, object] | str | None) -> str:+ """The file vocabulary stored in *page_metadata*, or ``""``."""+ if isinstance(page_metadata, str):+ try:+ page_metadata = json.loads(page_metadata or "{}")+ except ValueError:+ return ""+ if not isinstance(page_metadata, Mapping):+ return ""+ return str(page_metadata.get(FILE_VOCABULARY_KEY) or "")++def iter_embed_chunks(items: list[tuple[str, str, dict]],) -> Iterator[tuple[list[tuple[str, str, dict]], list[str]]]:diff --git a/packages/ui/src/graph/graph-doc-panel.tsx b/packages/ui/src/graph/graph-doc-panel.tsxindex c1eafc934..28306a6a4 100644--- a/packages/ui/src/graph/graph-doc-panel.tsx+++ b/packages/ui/src/graph/graph-doc-panel.tsx@@ -94,10 +94,12 @@ export function GraphDocPanel({status={page.freshness_status}showScore/>- <Badge variant="outline" className="font-mono text-[10px]">- <Cpu className="h-2.5 w-2.5 mr-0.5" />- {page.model_name}- </Badge>+ {page.model_name && (+ <Badge variant="outline" className="font-mono text-[10px]">+ <Cpu className="h-2.5 w-2.5 mr-0.5" />+ {page.model_name}+ </Badge>+ )}<span className="text-[10px] text-[var(--color-text-tertiary)] flex items-center gap-1 ml-auto"><Clock className="h-2.5 w-2.5" />{formatRelativeTime(page.updated_at)}diff --git a/tests/integration/test_deterministic_generation.py b/tests/integration/test_deterministic_generation.pyindex d175628a1..bc6bc1861 100644--- a/tests/integration/test_deterministic_generation.py+++ b/tests/integration/test_deterministic_generation.py@@ -126,7 +126,7 @@ class TestDeterministicGeneration:"""for p in deterministic_pages:assert (- "Built from the code itself" in p.content+ "Generated from parsed code, the import graph and git history" in p.contentor "Built from the code's structure" in p.content), p.page_iddiff --git a/tests/unit/cli/test_embedder_resolution.py b/tests/unit/cli/test_embedder_resolution.pyindex 06512b0fb..0088d71ee 100644--- a/tests/unit/cli/test_embedder_resolution.py+++ b/tests/unit/cli/test_embedder_resolution.py@@ -217,6 +217,7 @@ async def test_reindex_persists_its_resolved_embedder(# Non-nullable on the real model, so a stand-in that omits it is a# stand-in for a page that cannot exist.summary = "what a.py does"+ metadata_json = "{}"class _Result:def __init__(self, rows: list) -> None:diff --git a/tests/unit/cli/test_reindex_cmd.py b/tests/unit/cli/test_reindex_cmd.pyindex fa4a0e41f..88c74f860 100644--- a/tests/unit/cli/test_reindex_cmd.py+++ b/tests/unit/cli/test_reindex_cmd.py@@ -128,6 +128,7 @@ async def test_reindex_aborts_when_every_item_failed(target_path = "src/main.py"summary = ""content = "body"+ metadata_json = "{}"decision = "a decision"class _FailingSession:diff --git a/tests/unit/cli/test_reindex_embed_recipe.py b/tests/unit/cli/test_reindex_embed_recipe.pyindex 03349d312..f4ff02e4b 100644--- a/tests/unit/cli/test_reindex_embed_recipe.py+++ b/tests/unit/cli/test_reindex_embed_recipe.py@@ -23,6 +23,8 @@ class _DummyEngine:class _Page:+ metadata_json = "{}" # non-nullable on the real model+def __init__(self, **kw: Any) -> None:self.__dict__.update(kw)@@ -137,6 +139,22 @@ async def test_reindex_embeds_the_shared_recipe(run_reindex) -> None:]+async def test_reindex_embeds_the_stored_file_vocabulary(run_reindex) -> None:+ """The vocabulary is in ``metadata_json``, not the content, on a stored row."""+ page = _Page(+ id="file_page:pkg/list.go",+ title="File: pkg/list.go",+ page_type="file_page",+ target_path="pkg/list.go",+ summary="",+ content="## Overview\n\nLists releases.",+ metadata_json='{"file_vocabulary": "Order of releases returned"}',+ )++ store = await run_reindex([page])++ assert "Order of releases returned" in store.items[0][1]+async def test_reindex_carries_the_path_and_summary_into_the_text(run_reindex) -> None:"""The two fields it never used to pass, named directly.diff --git a/tests/unit/core/test_index_scope.py b/tests/unit/core/test_index_scope.pyindex b30e52f23..39044efb9 100644--- a/tests/unit/core/test_index_scope.py+++ b/tests/unit/core/test_index_scope.py@@ -49,3 +49,10 @@ def test_scope_surfaces_degraded_features_as_unavailable() -> None:"Execution flow: parser failed","health",]+++def test_mixed_provenance_survives_resolution() -> None:+ # A wiki where only some pages carry model prose is "mixed", not "model".+ state = {"run_mode": "standard", "docs_mode": "llm"}+ stamp_index_scope(state, {}, content_provenance="mixed")+ assert resolve_index_scope(state, {})["content_provenance"] == "mixed"diff --git a/tests/unit/generation/test_deterministic_templates.py b/tests/unit/generation/test_deterministic_templates.pyindex 68ae4c486..c9726c453 100644--- a/tests/unit/generation/test_deterministic_templates.py+++ b/tests/unit/generation/test_deterministic_templates.py@@ -531,9 +531,10 @@ def test_file_page_renders_german_headings_and_prose(german_generator):"## Wird verwendet von","## Nutzungshinweise","## Fragen, die diese Seite beantwortet",- "## Im Code",):assert heading in page.content, heading+ # The file vocabulary is embedded from metadata, never rendered.+ assert "## Im Code" not in page.contentassert "Sie stellt 1 öffentliches Symbol bereit" in page.contentassert "Importiert von 1 Datei in diesem Repository." in page.content# Identifiers are never translated, in any language.@@ -608,8 +609,8 @@ def test_the_footer_is_localized_on_every_structural_page(german_generator):template="file_page.j2",ctx=ctx,)- assert "Aus dem Code selbst erstellt" in page.content- assert "Built from the code itself" not in page.content+ assert "Erstellt aus geparstem Code" in page.content+ assert "Generated from parsed code" not in page.contentdef test_an_unsupported_language_renders_exactly_what_english_does(generator, klingon_generator):diff --git a/tests/unit/generation/test_file_vocabulary.py b/tests/unit/generation/test_file_vocabulary.pyindex 64774b651..64ca1b338 100644--- a/tests/unit/generation/test_file_vocabulary.py+++ b/tests/unit/generation/test_file_vocabulary.py@@ -9,15 +9,17 @@ Measured on the cli/cli and django canary indexes, gold file in the top ten ofa full-text search over page text: **Go 4 to 12 of 20 instances, python 34 to38 of 50**. The same probe against an index shaped like a symbol-onlycompetitor scored 8 of 20 on Go and *worse than the current pages* on python,-which is why this ships as page content and not as a second symbol index.+which is why this ships with the page and not as a second symbol index.-Two properties carry the whole result and each has a test below:+It is a bag of words, so it is stored in page metadata and embedded rather+than rendered: a reader saw it as token soup. Two properties carry the result+and each has a test below:-* it is **per-file**, so it discriminates. Repo-level vocabulary rendered here- would be byte-identical on every page in the corpus.-* it reaches the **full-text row**, not only the rendered page. The write path- hands ``page.content`` to the indexer verbatim, so nothing would raise if the- section were dropped or moved into metadata.+* it is **per-file**, so it discriminates. Repo-level vocabulary would be+ byte-identical on every page in the corpus.+* it reaches the **embedded text**, not only the page's metadata. The embed+ recipe reads it from there, so nothing would raise if a writer stopped+ handing the metadata over."""from __future__ import annotations@@ -26,16 +28,12 @@ from pathlib import Pathimport jinja2import pytest-from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine-from sqlalchemy.pool import StaticPoolfrom repowise.core.generation.context.file_vocabulary import file_vocabularyfrom repowise.core.generation.context_assembler import FilePageContextfrom repowise.core.generation.page_generator.structural import register_filtersfrom repowise.core.generation.structural_labels import resolve_structural_labels-from repowise.core.persistence.crud import upsert_page, upsert_repository-from repowise.core.persistence.database import init_db-from repowise.core.persistence.search import FullTextSearch+from repowise.core.persistence.vector_store import FILE_VOCABULARY_KEY, embed_itemVOCAB_HEADING = "## In the code"@@ -217,104 +215,42 @@ def _file_page(**overrides) -> FilePageContext:class TestRendering:- def test_the_section_renders_when_there_is_vocabulary(self, jinja_env):+ def test_the_vocabulary_is_not_rendered(self, jinja_env):ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))page = jinja_env.get_template("file_page.j2").render(ctx=ctx)- assert VOCAB_HEADING in page- assert "Order of releases returned" in page-- def test_no_heading_when_the_file_yields_nothing(self, jinja_env):- """Every section below the Overview is conditional, and this is why.-- An empty heading puts the same stock line into the index on thousands- of pages, where it matches every query and distinguishes none.- """- page = jinja_env.get_template("file_page.j2").render(ctx=_file_page(file_vocabulary=""))assert VOCAB_HEADING not in page+ assert "Order of releases returned" not in page- def test_it_sits_below_the_questions_block(self, jinja_env):- """``_extract_summary`` reads back from the top. A bag of words is not- a summary, so it must not be the first thing the page says."""- ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))- page = jinja_env.get_template("file_page.j2").render(ctx=ctx)- assert page.index("## Overview") < page.index(VOCAB_HEADING)- assert page.index("## Questions this page answers") < page.index(VOCAB_HEADING)- def test_two_files_get_different_vocabulary(self, jinja_env):- """The property the whole result rests on.+def test_two_files_get_different_vocabulary():+ """The property the whole result rests on.- Repo-level vocabulary would render byte-identically on all ~700 file- pages of cli/cli and carry zero discriminative power. This is the- opposite granularity, deliberately.- """- tmpl = jinja_env.get_template("file_page.j2")- a = tmpl.render(ctx=_file_page(file_vocabulary=file_vocabulary(GO_SOURCE)))- b = tmpl.render(- ctx=_file_page(- file_path="api/client.go",- file_vocabulary=file_vocabulary('type Client struct {\n\tHTTP string\n}'),- )- )- assert "Order of releases returned" in a- assert "Order of releases returned" not in b+ Repo-level vocabulary would be byte-identical on all ~700 file pages of+ cli/cli and carry zero discriminative power. This is the opposite+ granularity, deliberately.+ """+ a = file_vocabulary(GO_SOURCE)+ b = file_vocabulary("type Client struct {\n\tHTTP string\n}")+ assert "Order of releases returned" in a+ assert "Order of releases returned" not in b-async def fts():- engine = create_async_engine(- "sqlite+aiosqlite:///:memory:",- connect_args={"check_same_thread": False},- poolclass=StaticPool,- )- await init_db(engine)- search = FullTextSearch(engine)- await search.ensure_index()- yield engine, search- await engine.dispose()---async def _index(engine, search, *, page_id, target_path, content):- factory = async_sessionmaker(engine, expire_on_commit=False)- async with factory() as session:- repo = await upsert_repository(session, name="r", local_path="/tmp/r")- await session.commit()- await upsert_page(- session,- page_id=page_id,- repository_id=repo.id,- page_type="file_page",- title=target_path,- content=content,- summary="",- target_path=target_path,- source_hash="h",- model_name="template",- provider_name="template",- )- await session.commit()- await search.index(page_id, target_path, content, summary="", target_path=target_path)---async def test_the_words_reach_the_full_text_row(fts, jinja_env):+def test_the_words_reach_the_embedded_text(jinja_env):"""The round trip, not the render.- ``FullTextSearch.index`` is handed ``page.content`` verbatim, so the- section could be dropped, moved into metadata, or added after the row was- written and nothing would raise. This is the assertion that the page is- actually findable by the words its own file contains, which is the entire- claim.+ The vocabulary lives in page metadata, so a writer that passed only the+ content would embed a page without it and nothing would raise. This is the+ assertion that the embedded text carries the words the file itself uses."""- engine, search = ftsctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))content = jinja_env.get_template("file_page.j2").render(ctx=ctx)- await _index(- engine,- search,- page_id="file_page:pkg/cmd/release/list/list.go",+ _pid, text, _meta = embed_item(+ "file_page:pkg/cmd/release/list/list.go",+ title="pkg/cmd/release/list/list.go",+ page_type="file_page",target_path="pkg/cmd/release/list/list.go",+ summary="",content=content,+ page_metadata={FILE_VOCABULARY_KEY: ctx.file_vocabulary},)-- hits = await search.search("order of releases returned", limit=10)- assert [h.target_path for h in hits] == ["pkg/cmd/release/list/list.go"]+ assert "Order of releases returned" in textdiff --git a/tests/unit/generation/test_structural_pages_read_like_a_wiki.py b/tests/unit/generation/test_structural_pages_read_like_a_wiki.pyindex 6d2ba53d0..e5f04a66a 100644--- a/tests/unit/generation/test_structural_pages_read_like_a_wiki.py+++ b/tests/unit/generation/test_structural_pages_read_like_a_wiki.py@@ -367,7 +367,6 @@ def test_no_identifier_the_context_carried_leaves_the_page(generator):],kg_layer_name="Core Pipeline",kg_layer_role="edge_connector",- file_vocabulary="prune_dirs max_depth node_modules",)page = render(generator, ctx)@@ -377,7 +376,6 @@ def test_no_identifier_the_context_carried_leaves_the_page(generator):| set(ctx.dependents)| {s["name"] for s in ctx.symbols}| {cc["path"] for cc in ctx.co_change_pages}- | set(ctx.file_vocabulary.split()))missing = sorted(name for name in expected if name not in page)@@ -415,13 +413,15 @@ def _contributor_hide(source: str) -> set[str]:def test_the_scaffolding_headings_are_the_ones_the_reader_lens_hides(generator):"""The template and ``reader-persona.ts`` have to agree, byte for byte.- ``## In the code`` and ``## Questions this page answers`` exist for- retrieval: the vocabulary bag is the only part of the page written in the- words a question uses, and the question block is the only part shaped like- one. Neither can be dropped here without dropping it from the index, so- the reader persona hides them client-side instead — and it matches on the- heading text, lowercased, which is a string in another language's source- file. The headings come out of a real render for that reason.+ ``## Questions this page answers`` exists for retrieval: the question+ block is the only part of the page shaped like a question. It cannot be+ dropped here without dropping it from the index, so the reader persona+ hides it client-side instead — and it matches on the heading text,+ lowercased, which is a string in another language's source file. The+ heading comes out of a real render for that reason.++ ``## In the code`` is no longer rendered at all (the file vocabulary is+ embedded from page metadata), so the vocabulary never reaches a reader."""ctx = _context(dependents=["pkg/cli/main.py"],@@ -429,11 +429,12 @@ def test_the_scaffolding_headings_are_the_ones_the_reader_lens_hides(generator):)page = render(generator, ctx)rendered = {ln[3:].strip().lower() for ln in page.splitlines() if ln.startswith("## ")}- assert {"in the code", "questions this page answers"} <= rendered+ assert "questions this page answers" in rendered+ assert "in the code" not in renderedhidden = _contributor_hide(READER_PERSONA_TS.read_text(encoding="utf-8"))- assert {"in the code", "questions this page answers"} <= hidden+ assert "questions this page answers" in hidden# ---------------------------------------------------------------------------diff --git a/tests/unit/generation/test_wiki_template_cleanups.py b/tests/unit/generation/test_wiki_template_cleanups.pynew file mode 100644index 000000000..386973f65--- /dev/null+++ b/tests/unit/generation/test_wiki_template_cleanups.py@@ -0,0 +1,137 @@+"""What a wiki page says about itself, and what a model is told to write.++* The module prompt carries no dead-code findings: a "potentially unused"+ list handed to the model came back as confident prose about deletions.+* A file page's own vocabulary is embedded for search but not rendered:+ it lives in page metadata, and the one embed recipe reads it from there.+* The footer claims only what the page was built from.+* A template page credits no model; a model-written page names its model.+"""++from __future__ import annotations++import json++import networkx as nx+import pytest++from repowise.core.generation.context_assembler import ContextAssembler+from repowise.core.generation.models import GenerationConfig+from repowise.core.generation.page_generator import PageGenerator+from repowise.core.persistence.vector_store import FILE_VOCABULARY_KEY, embed_item+from repowise.core.providers.llm.mock import MockProvider++DEAD_CODE = [+ {+ "symbol_name": "legacy_resolver",+ "reason": "no callers found",+ "confidence": 0.9,+ "safe_to_delete": True,+ }+]++SOURCE = b'''\+class Calculator:+ """Adds two integers for the billing ledger."""++ def add(self, a: int, b: int) -> int:+ raise ValueError("ledger overflow while adding")+'''+++def _generator(provider=None, **config) -> PageGenerator:+ cfg = GenerationConfig(max_tokens=1024, token_budget=2000, max_concurrency=2, **config)+ return PageGenerator(provider or MockProvider(), ContextAssembler(cfg), cfg)+++async def _file_page(generator, parsed, graph, metrics):+ return await generator.generate_file_page(+ parsed,+ graph,+ metrics["pagerank"],+ metrics["betweenness"],+ metrics["community"],+ SOURCE,+ )+++async def test_the_module_prompt_has_no_dead_code_section():+ provider = MockProvider()+ await _generator(provider).generate_module_page(+ "Resolution Layer",+ "python",+ [],+ nx.DiGraph(),+ dead_code_findings=DEAD_CODE,+ target_path="core/resolvers",+ )++ prompt = provider.calls[-1]["user_prompt"]+ assert "Potentially unused code" not in prompt+ assert "legacy_resolver" not in prompt+++async def test_the_file_vocabulary_is_in_metadata_not_on_the_page(+ sample_parsed_file, sample_graph, graph_metrics+):+ page = await _file_page(_generator(), sample_parsed_file, sample_graph, graph_metrics)++ vocabulary = page.metadata[FILE_VOCABULARY_KEY]+ assert "ledger overflow while adding" in vocabulary+ assert "## In the code" not in page.content+ assert "ledger overflow while adding" not in page.content+++async def test_the_embed_recipe_picks_the_vocabulary_up(+ sample_parsed_file, sample_graph, graph_metrics+):+ """Both shapes a writer holds: a generated page's dict, a stored row's JSON."""+ page = await _file_page(_generator(), sample_parsed_file, sample_graph, graph_metrics)+ fields = dict(+ title=page.title,+ page_type=page.page_type,+ target_path=page.target_path,+ summary=page.summary,+ content=page.content,+ )++ for metadata in (page.metadata, json.dumps(page.metadata)):+ _pid, text, _meta = embed_item(page.page_id, page_metadata=metadata, **fields)+ assert "ledger overflow while adding" in text++ _pid, bare, _meta = embed_item(page.page_id, **fields)+ assert "ledger overflow while adding" not in bare+++ ("language", "footer", "retired"),+ [+ (None, "Generated from parsed code, the import graph and git history.", "checked against"),+ ("de", "Erstellt aus geparstem Code, dem Importgraphen und der Git-Historie.", "geprüft"),+ ],+)+async def test_the_footer_claims_only_its_inputs(+ language, footer, retired, sample_parsed_file, sample_graph, graph_metrics+):+ generator = _generator(language=language) if language else _generator()+ page = await _file_page(generator, sample_parsed_file, sample_graph, graph_metrics)++ assert footer in page.content+ assert retired not in page.content+++async def test_a_template_page_credits_no_model(sample_parsed_file, sample_graph, graph_metrics):+ generator = _generator(MockProvider(model="real-model-7"))+ page = await _file_page(generator, sample_parsed_file, sample_graph, graph_metrics)++ assert page.provider_name == "template"+ assert page.model_name == ""+++async def test_a_model_written_page_names_its_model():+ page = await _generator(MockProvider(model="real-model-7")).generate_module_page(+ "Resolution Layer", "python", [], nx.DiGraph(), target_path="core/resolvers"+ )++ assert page.provider_name == "mock"+ assert page.model_name == "real-model-7"