Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(wiki): drop dead code from the module prompt, keep file vocabulary out of the page, honest footer and model stamp (#2681)

Swati Ahuja65fcde37e5e6
diff --git a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
index 8e2ea4fdb..a622e9f65 100644
--- a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
+++ b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
@@ -881,6 +881,7 @@ def _run_repo_checks(
target_path=page.target_path or "",
summary=page.summary or "",
content=page.content or "",
+ page_metadata=page.metadata_json,
)
if item is None:
# Below the information floor, so its absence
diff --git a/packages/cli/src/repowise/cli/commands/reindex_cmd.py b/packages/cli/src/repowise/cli/commands/reindex_cmd.py
index adb5e2232..70b77e8db 100644
--- a/packages/cli/src/repowise/cli/commands/reindex_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/reindex_cmd.py
@@ -214,6 +214,7 @@ async def _reindex(repo_path, embedder_name: str, batch_size: int) -> None:
target_path=page.target_path or "",
summary=page.summary or "",
content=page.content or "",
+ page_metadata=page.metadata_json,
)
if item is None:
# Below the information floor. Not a failure — the page is
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
index 90ada550f..3cb54dff2 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
@@ -1659,6 +1659,7 @@ def run_update(
target_path=page.target_path,
summary=page.summary,
content=page.content,
+ page_metadata=page.metadata,
)
)
is not None
diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.py
index 82d30b14b..233a1752d 100644
--- a/packages/core/src/repowise/core/generation/page_generator/core.py
+++ b/packages/core/src/repowise/core/generation/page_generator/core.py
@@ -23,6 +23,7 @@ import jinja2
import structlog
from repowise.core.ingestion.models import ParsedFile, RepoStructure
+from repowise.core.persistence.vector_store import FILE_VOCABULARY_KEY
from repowise.core.providers.llm.base import BaseProvider, CacheHint, GeneratedResponse
from ..context.evidence import (
@@ -336,6 +337,9 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):
# file template does not emit; keep this type's original extraction.
page.summary = _extract_summary(page.content)
_attach_file_provenance(page, ctx)
+ # Embedded for search, not rendered (see file_page.j2).
+ if ctx.file_vocabulary:
+ page.metadata[FILE_VOCABULARY_KEY] = ctx.file_vocabulary
return page
# ------------------------------------------------------------------
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index 77f09289c..4ca635e10 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -1051,6 +1051,7 @@ def _embed_item(page: GeneratedPage) -> tuple[str, str, dict] | None:
target_path=page.target_path,
summary=page.summary,
content=page.content,
+ page_metadata=page.metadata,
)
diff --git a/packages/core/src/repowise/core/generation/page_generator/structural.py b/packages/core/src/repowise/core/generation/page_generator/structural.py
index dc97ea65b..16ec5dfdf 100644
--- a/packages/core/src/repowise/core/generation/page_generator/structural.py
+++ b/packages/core/src/repowise/core/generation/page_generator/structural.py
@@ -692,7 +692,7 @@ def stale_spotlight_paths(
class StructuralRenderMixin:
"""Template-only renderers, mixed into PageGenerator.
- Requires the host to provide ``_render``, ``_provider`` and ``_config``.
+ Requires the host to provide ``_render`` and ``_config``.
"""
def _render_page(
@@ -707,7 +707,8 @@ class StructuralRenderMixin:
"""Render one template page and wrap it as a GeneratedPage.
The mirror of ``_build_generated_page`` for the no-model path: same
- fields, zero tokens, ``provider_name="template"``.
+ fields, zero tokens, ``provider_name="template"``, and no model name:
+ no model wrote this page, so it must not be credited to one.
Confidence is a constant here, not a parameter. It used to be
overridable so a stub could claim less than a sole renderer's page,
@@ -727,7 +728,7 @@ class StructuralRenderMixin:
content=content,
summary=_extract_summary(content, skip_metadata=True),
source_hash=compute_source_hash(content),
- model_name=self._provider.model_name,
+ model_name="",
provider_name="template",
input_tokens=0,
output_tokens=0,
diff --git a/packages/core/src/repowise/core/generation/structural_labels.py b/packages/core/src/repowise/core/generation/structural_labels.py
index b27165978..fe73cc9aa 100644
--- a/packages/core/src/repowise/core/generation/structural_labels.py
+++ b/packages/core/src/repowise/core/generation/structural_labels.py
@@ -33,11 +33,7 @@ ENGLISH_LABELS: dict[str, str] = {
# -- shared ------------------------------------------------------------
"overview": "Overview",
"source": "Source",
- "footer": (
- "*Built from the code itself: parsed symbols, the import graph, git history and\n"
- "the knowledge graph. Every statement here is checked against the source rather\n"
- "than written about it.*"
- ),
+ "footer": "*Generated from parsed code, the import graph and git history.*",
"and_more": "and {count} more.",
"file": "File",
"file_singular": "file",
@@ -99,7 +95,6 @@ ENGLISH_LABELS: dict[str, str] = {
"changes_with_entry": "{count} shared {commit_word}",
"last_together": "last together on {date}",
"decisions_heading": "Decisions touching this file",
- "in_the_code": "In the code",
"question_exports": "What does `{path}` export?",
"question_where_defined": "Where is `{symbol}` defined?",
"question_what_imports": "What imports `{path}`?",
@@ -188,11 +183,7 @@ LOCALIZED_LABELS: dict[str, dict[str, str]] = {
"de": {
"overview": "Überblick",
"source": "Quelltext",
- "footer": (
- "*Aus dem Code selbst erstellt: geparste Symbole, der Importgraph, die "
- "Git-Historie\nund der Wissensgraph. Jede Aussage hier wird gegen den "
- "Quelltext geprüft, statt\nnur darüber geschrieben zu werden.*"
- ),
+ "footer": "*Erstellt aus geparstem Code, dem Importgraphen und der Git-Historie.*",
"and_more": "und {count} weitere.",
"file": "Datei",
"file_singular": "Datei",
@@ -247,7 +238,6 @@ LOCALIZED_LABELS: dict[str, dict[str, str]] = {
"changes_with_entry": "{count} gemeinsame {commit_word}",
"last_together": "zuletzt gemeinsam am {date}",
"decisions_heading": "Entscheidungen zu dieser Datei",
- "in_the_code": "Im Code",
"question_exports": "Was exportiert `{path}`?",
"question_where_defined": "Wo ist `{symbol}` definiert?",
"question_what_imports": "Was importiert `{path}`?",
diff --git a/packages/core/src/repowise/core/generation/templates/file_page.j2 b/packages/core/src/repowise/core/generation/templates/file_page.j2
index 4466e2673..b1b3e3d5a 100644
--- a/packages/core/src/repowise/core/generation/templates/file_page.j2
+++ b/packages/core/src/repowise/core/generation/templates/file_page.j2
@@ -206,29 +206,10 @@ _{{ labels.and_more.format(count=ctx.dependents | length - 25) }}_
- {{ q }}
{%- endfor %}
{%- endif %}
-{#- The words the file itself uses.
-
- Everything above is a description of the file. This is the file's own
- vocabulary: field names, string literals, help text, comment prose. The
- page could describe a file accurately and still be unfindable by any
- question about what it does, because a question is asked in the words of
- the source and none of those words were on the page.
-
- Per-file, and that is the whole point. Repo-level vocabulary rendered here
- would be byte-identical across every file page in the corpus, which is the
- failure this template's own header warns about: it would match every query
- and distinguish none of them.
-
- Last on the page, below the questions block, for the same reason that
- block is low: ``_extract_summary`` reads back from the top, and a bag of
- words is not a summary. Conditional, like every section below the
- Overview, so a file that yields no vocabulary renders no heading.
--#}
-{%- if ctx.file_vocabulary %}
-
-## {{ labels.in_the_code }}
-
-{{ ctx.file_vocabulary }}
-{%- endif %}
+{#- The file's own vocabulary (``ctx.file_vocabulary``) is not rendered.
+ ``_render_file_page`` stores it in page metadata and ``embed_item`` adds
+ it to the embedded text: it is a bag of words, which serves retrieval and
+ reads to a person as noise.
+#}
{% include "_footer.j2" %}
diff --git a/packages/core/src/repowise/core/generation/templates/module_page.j2 b/packages/core/src/repowise/core/generation/templates/module_page.j2
index e0e58bfdd..754c1a47c 100644
--- a/packages/core/src/repowise/core/generation/templates/module_page.j2
+++ b/packages/core/src/repowise/core/generation/templates/module_page.j2
@@ -109,13 +109,6 @@ These modules change together with this one in history and nothing in the depend
{% endfor %}
{% endif %}
-{% if ctx.dead_code_findings %}
-### Potentially unused code
-{% for finding in ctx.dead_code_findings[:10] %}
-- `{{ finding.symbol_name or '?' }}` — {{ finding.reason }} ({{ (finding.confidence * 100) | int }}%{% if finding.safe_to_delete %}, cleanup-ready{% endif %})
-{% endfor %}
-{% endif %}
-
{% if module_git_summary is defined and module_git_summary %}
### Recent activity
Most active file: `{{ module_git_summary.most_active_file }}` ({{ module_git_summary.most_active_commits_90d }} commits in 90 days).
diff --git a/packages/core/src/repowise/core/index_scope.py b/packages/core/src/repowise/core/index_scope.py
index 5ce09fa14..4b300529c 100644
--- a/packages/core/src/repowise/core/index_scope.py
+++ b/packages/core/src/repowise/core/index_scope.py
@@ -114,7 +114,7 @@ def resolve_index_scope(
git_tier = _choice(state.get("git_tier", scope.get("git_tier")), {"essential", "full"})
if isinstance(stored, Mapping):
provenance = _choice(
- scope.get("content_provenance", provenance), {"none", "template", "model"}
+ scope.get("content_provenance", provenance), {"none", "template", "model", "mixed"}
)
unavailable = analysis.get("unavailable", [])
diff --git a/packages/core/src/repowise/core/persistence/vector_store/__init__.py b/packages/core/src/repowise/core/persistence/vector_store/__init__.py
index f83f059d6..10070ea5b 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/__init__.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/__init__.py
@@ -27,6 +27,7 @@ to avoid one embedding round-trip per page; the single-item
from __future__ import annotations
from ._base import (
+ FILE_VOCABULARY_KEY,
BatchChunkFailure,
BatchEmbeddingError,
VectorStore,
@@ -39,6 +40,7 @@ from .lancedb_store import LanceDBVectorStore
from .pgvector_store import PgVectorStore
__all__ = [
+ "FILE_VOCABULARY_KEY",
"BatchChunkFailure",
"BatchEmbeddingError",
"InMemoryVectorStore",
diff --git a/packages/core/src/repowise/core/persistence/vector_store/_base.py b/packages/core/src/repowise/core/persistence/vector_store/_base.py
index acc6b31fb..bbce123c8 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/_base.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/_base.py
@@ -8,10 +8,11 @@ re-exported from the package ``__init__`` so the historical import path
from __future__ import annotations
+import json
import logging
import math
from abc import ABC, abstractmethod
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from typing import Literal
@@ -21,6 +22,7 @@ from ..search import SearchResult
__all__ = [
"EMBED_BATCH_MAX_ITEMS",
"EMBED_TEXT_MAX_CHARS",
+ "FILE_VOCABULARY_KEY",
"STORED_SNIPPET_CHARS",
"BatchChunkFailure",
"BatchEmbeddingError",
@@ -95,6 +97,11 @@ EMBED_TEXT_MAX_CHARS = 16_000
# and on the paths that passed no content at all, an empty string.
STORED_SNIPPET_CHARS = 2_000
+# Page-metadata key for a file page's own vocabulary (field names, string
+# literals, comment prose). It is embedded with the page but not rendered on
+# it: a bag of words helps retrieval and reads to a person as noise.
+FILE_VOCABULARY_KEY = "file_vocabulary"
+
def cap_embed_text(page_id: str, text: str) -> str:
"""Return text bounded for one embedding input and report any loss.
@@ -126,6 +133,7 @@ def embed_item(
target_path: str,
summary: str,
content: str,
+ page_metadata: Mapping[str, object] | str | None = None,
) -> tuple[str, str, dict] | None:
"""Build the one ``(page_id, text, metadata)`` item every writer embeds.
@@ -164,6 +172,11 @@ def embed_item(
resolves as a link target, and a reader who arrives at it still learns the
file exists. It is only kept out of the index, where its cost is paid by
other pages. The floor is 0 by default, which admits everything.
+
+ ``page_metadata`` is the page's metadata, as a dict (a generated page) or
+ the stored JSON string (a ``wiki_pages`` row). Only
+ :data:`FILE_VOCABULARY_KEY` is read from it, and appended after the
+ content; the floor still measures ``content`` alone.
"""
if not title.strip():
raise ValueError(
@@ -174,7 +187,7 @@ def embed_item(
if not meets_information_floor(content):
count_page_denied_a_vector()
return None
- parts = [p for p in (title, target_path, summary, content) if p]
+ parts = [p for p in (title, target_path, summary, content, _vocabulary(page_metadata)) if p]
return (
page_id,
"\n".join(parts),
@@ -190,6 +203,18 @@ def embed_item(
)
+def _vocabulary(page_metadata: Mapping[str, object] | str | None) -> str:
+ """The file vocabulary stored in *page_metadata*, or ``""``."""
+ if isinstance(page_metadata, str):
+ try:
+ page_metadata = json.loads(page_metadata or "{}")
+ except ValueError:
+ return ""
+ if not isinstance(page_metadata, Mapping):
+ return ""
+ return str(page_metadata.get(FILE_VOCABULARY_KEY) or "")
+
+
def iter_embed_chunks(
items: list[tuple[str, str, dict]],
) -> Iterator[tuple[list[tuple[str, str, dict]], list[str]]]:
diff --git a/packages/ui/src/graph/graph-doc-panel.tsx b/packages/ui/src/graph/graph-doc-panel.tsx
index c1eafc934..28306a6a4 100644
--- a/packages/ui/src/graph/graph-doc-panel.tsx
+++ b/packages/ui/src/graph/graph-doc-panel.tsx
@@ -94,10 +94,12 @@ export function GraphDocPanel({
status={page.freshness_status}
showScore
/>
- <Badge variant="outline" className="font-mono text-[10px]">
- <Cpu className="h-2.5 w-2.5 mr-0.5" />
- {page.model_name}
- </Badge>
+ {page.model_name && (
+ <Badge variant="outline" className="font-mono text-[10px]">
+ <Cpu className="h-2.5 w-2.5 mr-0.5" />
+ {page.model_name}
+ </Badge>
+ )}
<span className="text-[10px] text-[var(--color-text-tertiary)] flex items-center gap-1 ml-auto">
<Clock className="h-2.5 w-2.5" />
{formatRelativeTime(page.updated_at)}
diff --git a/tests/integration/test_deterministic_generation.py b/tests/integration/test_deterministic_generation.py
index d175628a1..bc6bc1861 100644
--- a/tests/integration/test_deterministic_generation.py
+++ b/tests/integration/test_deterministic_generation.py
@@ -126,7 +126,7 @@ class TestDeterministicGeneration:
"""
for p in deterministic_pages:
assert (
- "Built from the code itself" in p.content
+ "Generated from parsed code, the import graph and git history" in p.content
or "Built from the code's structure" in p.content
), p.page_id
diff --git a/tests/unit/cli/test_embedder_resolution.py b/tests/unit/cli/test_embedder_resolution.py
index 06512b0fb..0088d71ee 100644
--- a/tests/unit/cli/test_embedder_resolution.py
+++ b/tests/unit/cli/test_embedder_resolution.py
@@ -217,6 +217,7 @@ async def test_reindex_persists_its_resolved_embedder(
# Non-nullable on the real model, so a stand-in that omits it is a
# stand-in for a page that cannot exist.
summary = "what a.py does"
+ metadata_json = "{}"
class _Result:
def __init__(self, rows: list) -> None:
diff --git a/tests/unit/cli/test_reindex_cmd.py b/tests/unit/cli/test_reindex_cmd.py
index fa4a0e41f..88c74f860 100644
--- a/tests/unit/cli/test_reindex_cmd.py
+++ b/tests/unit/cli/test_reindex_cmd.py
@@ -128,6 +128,7 @@ async def test_reindex_aborts_when_every_item_failed(
target_path = "src/main.py"
summary = ""
content = "body"
+ metadata_json = "{}"
decision = "a decision"
class _FailingSession:
diff --git a/tests/unit/cli/test_reindex_embed_recipe.py b/tests/unit/cli/test_reindex_embed_recipe.py
index 03349d312..f4ff02e4b 100644
--- a/tests/unit/cli/test_reindex_embed_recipe.py
+++ b/tests/unit/cli/test_reindex_embed_recipe.py
@@ -23,6 +23,8 @@ class _DummyEngine:
class _Page:
+ metadata_json = "{}" # non-nullable on the real model
+
def __init__(self, **kw: Any) -> None:
self.__dict__.update(kw)
@@ -137,6 +139,22 @@ async def test_reindex_embeds_the_shared_recipe(run_reindex) -> None:
]
+async def test_reindex_embeds_the_stored_file_vocabulary(run_reindex) -> None:
+ """The vocabulary is in ``metadata_json``, not the content, on a stored row."""
+ page = _Page(
+ id="file_page:pkg/list.go",
+ title="File: pkg/list.go",
+ page_type="file_page",
+ target_path="pkg/list.go",
+ summary="",
+ content="## Overview\n\nLists releases.",
+ metadata_json='{"file_vocabulary": "Order of releases returned"}',
+ )
+
+ store = await run_reindex([page])
+
+ assert "Order of releases returned" in store.items[0][1]
+
async def test_reindex_carries_the_path_and_summary_into_the_text(run_reindex) -> None:
"""The two fields it never used to pass, named directly.
diff --git a/tests/unit/core/test_index_scope.py b/tests/unit/core/test_index_scope.py
index b30e52f23..39044efb9 100644
--- a/tests/unit/core/test_index_scope.py
+++ b/tests/unit/core/test_index_scope.py
@@ -49,3 +49,10 @@ def test_scope_surfaces_degraded_features_as_unavailable() -> None:
"Execution flow: parser failed",
"health",
]
+
+
+def test_mixed_provenance_survives_resolution() -> None:
+ # A wiki where only some pages carry model prose is "mixed", not "model".
+ state = {"run_mode": "standard", "docs_mode": "llm"}
+ stamp_index_scope(state, {}, content_provenance="mixed")
+ assert resolve_index_scope(state, {})["content_provenance"] == "mixed"
diff --git a/tests/unit/generation/test_deterministic_templates.py b/tests/unit/generation/test_deterministic_templates.py
index 68ae4c486..c9726c453 100644
--- a/tests/unit/generation/test_deterministic_templates.py
+++ b/tests/unit/generation/test_deterministic_templates.py
@@ -531,9 +531,10 @@ def test_file_page_renders_german_headings_and_prose(german_generator):
"## Wird verwendet von",
"## Nutzungshinweise",
"## Fragen, die diese Seite beantwortet",
- "## Im Code",
):
assert heading in page.content, heading
+ # The file vocabulary is embedded from metadata, never rendered.
+ assert "## Im Code" not in page.content
assert "Sie stellt 1 öffentliches Symbol bereit" in page.content
assert "Importiert von 1 Datei in diesem Repository." in page.content
# Identifiers are never translated, in any language.
@@ -608,8 +609,8 @@ def test_the_footer_is_localized_on_every_structural_page(german_generator):
template="file_page.j2",
ctx=ctx,
)
- assert "Aus dem Code selbst erstellt" in page.content
- assert "Built from the code itself" not in page.content
+ assert "Erstellt aus geparstem Code" in page.content
+ assert "Generated from parsed code" not in page.content
def test_an_unsupported_language_renders_exactly_what_english_does(generator, klingon_generator):
diff --git a/tests/unit/generation/test_file_vocabulary.py b/tests/unit/generation/test_file_vocabulary.py
index 64774b651..64ca1b338 100644
--- a/tests/unit/generation/test_file_vocabulary.py
+++ b/tests/unit/generation/test_file_vocabulary.py
@@ -9,15 +9,17 @@ Measured on the cli/cli and django canary indexes, gold file in the top ten of
a full-text search over page text: **Go 4 to 12 of 20 instances, python 34 to
38 of 50**. The same probe against an index shaped like a symbol-only
competitor scored 8 of 20 on Go and *worse than the current pages* on python,
-which is why this ships as page content and not as a second symbol index.
+which is why this ships with the page and not as a second symbol index.
-Two properties carry the whole result and each has a test below:
+It is a bag of words, so it is stored in page metadata and embedded rather
+than rendered: a reader saw it as token soup. Two properties carry the result
+and each has a test below:
-* it is **per-file**, so it discriminates. Repo-level vocabulary rendered here
- would be byte-identical on every page in the corpus.
-* it reaches the **full-text row**, not only the rendered page. The write path
- hands ``page.content`` to the indexer verbatim, so nothing would raise if the
- section were dropped or moved into metadata.
+* it is **per-file**, so it discriminates. Repo-level vocabulary would be
+ byte-identical on every page in the corpus.
+* it reaches the **embedded text**, not only the page's metadata. The embed
+ recipe reads it from there, so nothing would raise if a writer stopped
+ handing the metadata over.
"""
from __future__ import annotations
@@ -26,16 +28,12 @@ from pathlib import Path
import jinja2
import pytest
-from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
-from sqlalchemy.pool import StaticPool
from repowise.core.generation.context.file_vocabulary import file_vocabulary
from repowise.core.generation.context_assembler import FilePageContext
from repowise.core.generation.page_generator.structural import register_filters
from repowise.core.generation.structural_labels import resolve_structural_labels
-from repowise.core.persistence.crud import upsert_page, upsert_repository
-from repowise.core.persistence.database import init_db
-from repowise.core.persistence.search import FullTextSearch
+from repowise.core.persistence.vector_store import FILE_VOCABULARY_KEY, embed_item
VOCAB_HEADING = "## In the code"
@@ -217,104 +215,42 @@ def _file_page(**overrides) -> FilePageContext:
class TestRendering:
- def test_the_section_renders_when_there_is_vocabulary(self, jinja_env):
+ def test_the_vocabulary_is_not_rendered(self, jinja_env):
ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))
page = jinja_env.get_template("file_page.j2").render(ctx=ctx)
- assert VOCAB_HEADING in page
- assert "Order of releases returned" in page
-
- def test_no_heading_when_the_file_yields_nothing(self, jinja_env):
- """Every section below the Overview is conditional, and this is why.
-
- An empty heading puts the same stock line into the index on thousands
- of pages, where it matches every query and distinguishes none.
- """
- page = jinja_env.get_template("file_page.j2").render(ctx=_file_page(file_vocabulary=""))
assert VOCAB_HEADING not in page
+ assert "Order of releases returned" not in page
- def test_it_sits_below_the_questions_block(self, jinja_env):
- """``_extract_summary`` reads back from the top. A bag of words is not
- a summary, so it must not be the first thing the page says."""
- ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))
- page = jinja_env.get_template("file_page.j2").render(ctx=ctx)
- assert page.index("## Overview") < page.index(VOCAB_HEADING)
- assert page.index("## Questions this page answers") < page.index(VOCAB_HEADING)
- def test_two_files_get_different_vocabulary(self, jinja_env):
- """The property the whole result rests on.
+def test_two_files_get_different_vocabulary():
+ """The property the whole result rests on.
- Repo-level vocabulary would render byte-identically on all ~700 file
- pages of cli/cli and carry zero discriminative power. This is the
- opposite granularity, deliberately.
- """
- tmpl = jinja_env.get_template("file_page.j2")
- a = tmpl.render(ctx=_file_page(file_vocabulary=file_vocabulary(GO_SOURCE)))
- b = tmpl.render(
- ctx=_file_page(
- file_path="api/client.go",
- file_vocabulary=file_vocabulary('type Client struct {\n\tHTTP string\n}'),
- )
- )
- assert "Order of releases returned" in a
- assert "Order of releases returned" not in b
+ Repo-level vocabulary would be byte-identical on all ~700 file pages of
+ cli/cli and carry zero discriminative power. This is the opposite
+ granularity, deliberately.
+ """
+ a = file_vocabulary(GO_SOURCE)
+ b = file_vocabulary("type Client struct {\n\tHTTP string\n}")
+ assert "Order of releases returned" in a
+ assert "Order of releases returned" not in b
-async def fts():
- engine = create_async_engine(
- "sqlite+aiosqlite:///:memory:",
- connect_args={"check_same_thread": False},
- poolclass=StaticPool,
- )
- await init_db(engine)
- search = FullTextSearch(engine)
- await search.ensure_index()
- yield engine, search
- await engine.dispose()
-
-
-async def _index(engine, search, *, page_id, target_path, content):
- factory = async_sessionmaker(engine, expire_on_commit=False)
- async with factory() as session:
- repo = await upsert_repository(session, name="r", local_path="/tmp/r")
- await session.commit()
- await upsert_page(
- session,
- page_id=page_id,
- repository_id=repo.id,
- page_type="file_page",
- title=target_path,
- content=content,
- summary="",
- target_path=target_path,
- source_hash="h",
- model_name="template",
- provider_name="template",
- )
- await session.commit()
- await search.index(page_id, target_path, content, summary="", target_path=target_path)
-
-
-async def test_the_words_reach_the_full_text_row(fts, jinja_env):
+def test_the_words_reach_the_embedded_text(jinja_env):
"""The round trip, not the render.
- ``FullTextSearch.index`` is handed ``page.content`` verbatim, so the
- section could be dropped, moved into metadata, or added after the row was
- written and nothing would raise. This is the assertion that the page is
- actually findable by the words its own file contains, which is the entire
- claim.
+ The vocabulary lives in page metadata, so a writer that passed only the
+ content would embed a page without it and nothing would raise. This is the
+ assertion that the embedded text carries the words the file itself uses.
"""
- engine, search = fts
ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))
content = jinja_env.get_template("file_page.j2").render(ctx=ctx)
- await _index(
- engine,
- search,
- page_id="file_page:pkg/cmd/release/list/list.go",
+ _pid, text, _meta = embed_item(
+ "file_page:pkg/cmd/release/list/list.go",
+ title="pkg/cmd/release/list/list.go",
+ page_type="file_page",
target_path="pkg/cmd/release/list/list.go",
+ summary="",
content=content,
+ page_metadata={FILE_VOCABULARY_KEY: ctx.file_vocabulary},
)
-
- hits = await search.search("order of releases returned", limit=10)
- assert [h.target_path for h in hits] == ["pkg/cmd/release/list/list.go"]
+ assert "Order of releases returned" in text
diff --git a/tests/unit/generation/test_structural_pages_read_like_a_wiki.py b/tests/unit/generation/test_structural_pages_read_like_a_wiki.py
index 6d2ba53d0..e5f04a66a 100644
--- a/tests/unit/generation/test_structural_pages_read_like_a_wiki.py
+++ b/tests/unit/generation/test_structural_pages_read_like_a_wiki.py
@@ -367,7 +367,6 @@ def test_no_identifier_the_context_carried_leaves_the_page(generator):
],
kg_layer_name="Core Pipeline",
kg_layer_role="edge_connector",
- file_vocabulary="prune_dirs max_depth node_modules",
)
page = render(generator, ctx)
@@ -377,7 +376,6 @@ def test_no_identifier_the_context_carried_leaves_the_page(generator):
| set(ctx.dependents)
| {s["name"] for s in ctx.symbols}
| {cc["path"] for cc in ctx.co_change_pages}
- | set(ctx.file_vocabulary.split())
)
missing = sorted(name for name in expected if name not in page)
@@ -415,13 +413,15 @@ def _contributor_hide(source: str) -> set[str]:
def test_the_scaffolding_headings_are_the_ones_the_reader_lens_hides(generator):
"""The template and ``reader-persona.ts`` have to agree, byte for byte.
- ``## In the code`` and ``## Questions this page answers`` exist for
- retrieval: the vocabulary bag is the only part of the page written in the
- words a question uses, and the question block is the only part shaped like
- one. Neither can be dropped here without dropping it from the index, so
- the reader persona hides them client-side instead — and it matches on the
- heading text, lowercased, which is a string in another language's source
- file. The headings come out of a real render for that reason.
+ ``## Questions this page answers`` exists for retrieval: the question
+ block is the only part of the page shaped like a question. It cannot be
+ dropped here without dropping it from the index, so the reader persona
+ hides it client-side instead — and it matches on the heading text,
+ lowercased, which is a string in another language's source file. The
+ heading comes out of a real render for that reason.
+
+ ``## In the code`` is no longer rendered at all (the file vocabulary is
+ embedded from page metadata), so the vocabulary never reaches a reader.
"""
ctx = _context(
dependents=["pkg/cli/main.py"],
@@ -429,11 +429,12 @@ def test_the_scaffolding_headings_are_the_ones_the_reader_lens_hides(generator):
)
page = render(generator, ctx)
rendered = {ln[3:].strip().lower() for ln in page.splitlines() if ln.startswith("## ")}
- assert {"in the code", "questions this page answers"} <= rendered
+ assert "questions this page answers" in rendered
+ assert "in the code" not in rendered
hidden = _contributor_hide(READER_PERSONA_TS.read_text(encoding="utf-8"))
- assert {"in the code", "questions this page answers"} <= hidden
+ assert "questions this page answers" in hidden
# ---------------------------------------------------------------------------
diff --git a/tests/unit/generation/test_wiki_template_cleanups.py b/tests/unit/generation/test_wiki_template_cleanups.py
new file mode 100644
index 000000000..386973f65
--- /dev/null
+++ b/tests/unit/generation/test_wiki_template_cleanups.py
@@ -0,0 +1,137 @@
+"""What a wiki page says about itself, and what a model is told to write.
+
+* The module prompt carries no dead-code findings: a "potentially unused"
+ list handed to the model came back as confident prose about deletions.
+* A file page's own vocabulary is embedded for search but not rendered:
+ it lives in page metadata, and the one embed recipe reads it from there.
+* The footer claims only what the page was built from.
+* A template page credits no model; a model-written page names its model.
+"""
+
+from __future__ import annotations
+
+import json
+
+import networkx as nx
+import pytest
+
+from repowise.core.generation.context_assembler import ContextAssembler
+from repowise.core.generation.models import GenerationConfig
+from repowise.core.generation.page_generator import PageGenerator
+from repowise.core.persistence.vector_store import FILE_VOCABULARY_KEY, embed_item
+from repowise.core.providers.llm.mock import MockProvider
+
+DEAD_CODE = [
+ {
+ "symbol_name": "legacy_resolver",
+ "reason": "no callers found",
+ "confidence": 0.9,
+ "safe_to_delete": True,
+ }
+]
+
+SOURCE = b'''\
+class Calculator:
+ """Adds two integers for the billing ledger."""
+
+ def add(self, a: int, b: int) -> int:
+ raise ValueError("ledger overflow while adding")
+'''
+
+
+def _generator(provider=None, **config) -> PageGenerator:
+ cfg = GenerationConfig(max_tokens=1024, token_budget=2000, max_concurrency=2, **config)
+ return PageGenerator(provider or MockProvider(), ContextAssembler(cfg), cfg)
+
+
+async def _file_page(generator, parsed, graph, metrics):
+ return await generator.generate_file_page(
+ parsed,
+ graph,
+ metrics["pagerank"],
+ metrics["betweenness"],
+ metrics["community"],
+ SOURCE,
+ )
+
+
+async def test_the_module_prompt_has_no_dead_code_section():
+ provider = MockProvider()
+ await _generator(provider).generate_module_page(
+ "Resolution Layer",
+ "python",
+ [],
+ nx.DiGraph(),
+ dead_code_findings=DEAD_CODE,
+ target_path="core/resolvers",
+ )
+
+ prompt = provider.calls[-1]["user_prompt"]
+ assert "Potentially unused code" not in prompt
+ assert "legacy_resolver" not in prompt
+
+
+async def test_the_file_vocabulary_is_in_metadata_not_on_the_page(
+ sample_parsed_file, sample_graph, graph_metrics
+):
+ page = await _file_page(_generator(), sample_parsed_file, sample_graph, graph_metrics)
+
+ vocabulary = page.metadata[FILE_VOCABULARY_KEY]
+ assert "ledger overflow while adding" in vocabulary
+ assert "## In the code" not in page.content
+ assert "ledger overflow while adding" not in page.content
+
+
+async def test_the_embed_recipe_picks_the_vocabulary_up(
+ sample_parsed_file, sample_graph, graph_metrics
+):
+ """Both shapes a writer holds: a generated page's dict, a stored row's JSON."""
+ page = await _file_page(_generator(), sample_parsed_file, sample_graph, graph_metrics)
+ fields = dict(
+ title=page.title,
+ page_type=page.page_type,
+ target_path=page.target_path,
+ summary=page.summary,
+ content=page.content,
+ )
+
+ for metadata in (page.metadata, json.dumps(page.metadata)):
+ _pid, text, _meta = embed_item(page.page_id, page_metadata=metadata, **fields)
+ assert "ledger overflow while adding" in text
+
+ _pid, bare, _meta = embed_item(page.page_id, **fields)
+ assert "ledger overflow while adding" not in bare
+
+
+ ("language", "footer", "retired"),
+ [
+ (None, "Generated from parsed code, the import graph and git history.", "checked against"),
+ ("de", "Erstellt aus geparstem Code, dem Importgraphen und der Git-Historie.", "geprüft"),
+ ],
+)
+async def test_the_footer_claims_only_its_inputs(
+ language, footer, retired, sample_parsed_file, sample_graph, graph_metrics
+):
+ generator = _generator(language=language) if language else _generator()
+ page = await _file_page(generator, sample_parsed_file, sample_graph, graph_metrics)
+
+ assert footer in page.content
+ assert retired not in page.content
+
+
+async def test_a_template_page_credits_no_model(sample_parsed_file, sample_graph, graph_metrics):
+ generator = _generator(MockProvider(model="real-model-7"))
+ page = await _file_page(generator, sample_parsed_file, sample_graph, graph_metrics)
+
+ assert page.provider_name == "template"
+ assert page.model_name == ""
+
+
+async def test_a_model_written_page_names_its_model():
+ page = await _generator(MockProvider(model="real-model-7")).generate_module_page(
+ "Resolution Layer", "python", [], nx.DiGraph(), target_path="core/resolvers"
+ )
+
+ assert page.provider_name == "mock"
+ assert page.model_name == "real-model-7"