Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): file pages and spotlights name the questions they answer (#1209)
Swati Ahujaef660a4cac7c
diff --git a/packages/core/src/repowise/core/generation/report.py b/packages/core/src/repowise/core/generation/report.pyindex 2b71af27b..f13c8a33a 100644--- a/packages/core/src/repowise/core/generation/report.py+++ b/packages/core/src/repowise/core/generation/report.py@@ -20,6 +20,30 @@ from .prose import prose_word_count# on purpose — a long overview is worth seeing, never worth failing a run over.ORIENTATION_PROSE_WORD_BUDGET = 450+# The heading the deterministic templates put their question-shaped text under.+# Counted rather than asserted: the block is conditional by design, so a page+# without one is legitimate and only the ratio is meaningful.+QUESTIONS_HEADING = "## Questions this page answers"++# Page types rendered from a template with no model path, and therefore the+# ones that carry a deterministic questions block.+_QUESTION_PAGE_TYPES = ("file_page", "symbol_spotlight")+++def _count_question_blocks(pages: list[GeneratedPage]) -> dict[str, int]:+ """How many template pages carry question-shaped text, out of how many.++ Both numbers, because the ratio is the only reading that means anything.+ A zero numerator on a zero denominator is a run that wrote no such pages;+ a zero numerator on a large denominator is the block having silently+ stopped rendering, which nothing else in the run would report.+ """+ eligible = [p for p in pages if p.page_type in _QUESTION_PAGE_TYPES]+ return {+ "eligible_pages": len(eligible),+ "with_questions": sum(1 for p in eligible if QUESTIONS_HEADING in (p.content or "")),+ }+@dataclassclass GenerationReport:@@ -58,6 +82,12 @@ class GenerationReport:# kept and still resolves as a link target; only its vector is withheld.# Zero whenever the information floor is off, which is the default.pages_denied_a_vector: int = 0+ # How far question-shaped text reached across this run's template-rendered+ # pages. Carries its own denominator, so "this run wrote no such pages"+ # and "the block stopped rendering" stay separate facts. Without the+ # denominator a silent template regression is indistinguishable from a run+ # that generated no file pages, and neither one raises.+ question_blocks: dict[str, int] = field(default_factory=dict)@classmethoddef from_pages(@@ -92,6 +122,7 @@ class GenerationReport:layer_grouping=measure_layer_grouping(pages),artifact_checks=artifact_check_counts(),pages_denied_a_vector=pages_denied_a_vector(),+ question_blocks=_count_question_blocks(pages),overview_prose_words=next((prose_word_count(p.content or "")@@ -113,6 +144,19 @@ class GenerationReport:return Falsereturn self.overview_prose_words > ORIENTATION_PROSE_WORD_BUDGET+ @property+ def questions_missing(self) -> bool:+ """Whether a template page this run wrote came out with no questions.++ Warn-only, and expected to be non-zero: a file with no symbols and no+ resolved edges has nothing structural to ask about, so it legitimately+ renders none. What the flag is for is the other case — the number+ falling to zero, or near it, across a run that wrote thousands of+ pages, which is what a broken template looks like from outside.+ """+ eligible = self.question_blocks.get("eligible_pages", 0)+ return bool(eligible) and self.question_blocks.get("with_questions", 0) < eligible+def overview_length_summary(self) -> str:"""One line naming the overview's length against its budget."""if self.overview_prose_words is None:@@ -120,6 +164,13 @@ class GenerationReport:line = f"{self.overview_prose_words} / {ORIENTATION_PROSE_WORD_BUDGET} words"return f"{line} — over budget" if self.overview_over_budget else line+ def question_block_summary(self) -> str:+ """One line saying how far question-shaped text reached this run."""+ eligible = self.question_blocks.get("eligible_pages", 0)+ if not eligible:+ return "not measured (0 template pages)"+ return f"{self.question_blocks.get('with_questions', 0)} of {eligible} pages"+def artifact_check_summary(self) -> str:"""One line saying whether the artifact checks ran, and what they found."""checked = self.artifact_checks.get("responses_checked", 0)@@ -213,6 +264,11 @@ def render_generation_checks(report: GenerationReport, console: object) -> None:length_text = f"[yellow]{length_text}[/yellow]"table.add_row("Overview length", length_text)+ questions_text = report.question_block_summary()+ if report.questions_missing:+ questions_text = f"[yellow]{questions_text}[/yellow]"+ table.add_row("Question-shaped text", questions_text)+console.print(table) # type: ignore[union-attr]for pair in overlap.flagged:diff --git a/tests/unit/generation/test_question_block_report.py b/tests/unit/generation/test_question_block_report.pynew file mode 100644index 000000000..94c65608e--- /dev/null+++ b/tests/unit/generation/test_question_block_report.py@@ -0,0 +1,102 @@+"""The run reports how far question-shaped text reached.++The block is rendered by a template, so nothing raises when it stops being+rendered — a regression would look like an ordinary successful run producing+slightly shorter pages, and the only symptom would be retrieval quietly+getting worse on the next index. This counter is what makes that visible.++It carries its own denominator because the numerator alone cannot be read: a+zero is either "this run wrote no template pages" or "the block is broken",+and those need different responses.+"""++from __future__ import annotations++from repowise.core.generation.models import GeneratedPage+from repowise.core.generation.report import QUESTIONS_HEADING, GenerationReport++_WITH = f"# A file\n\n## Overview\n\nText.\n\n{QUESTIONS_HEADING}\n\n- What imports `a.py`?\n"+_WITHOUT = "# A file\n\n## Overview\n\nText.\n"+++def _page(page_id: str, page_type: str, content: str) -> GeneratedPage:+ return GeneratedPage(+ page_id=page_id,+ page_type=page_type,+ title=page_id,+ content=content,+ source_hash="",+ model_name="template",+ provider_name="template",+ input_tokens=0,+ output_tokens=0,+ cached_tokens=0,+ generation_level=2,+ target_path="",+ created_at="2026-07-31T00:00:00Z",+ updated_at="2026-07-31T00:00:00Z",+ )+++def test_the_report_counts_pages_carrying_questions():+ report = GenerationReport.from_pages(+ [+ _page("file_page:a.py", "file_page", _WITH),+ _page("file_page:b.py", "file_page", _WITHOUT),+ _page("symbol_spotlight:a.py::F", "symbol_spotlight", _WITH),+ ]+ )++ assert report.question_blocks == {"eligible_pages": 3, "with_questions": 2}+ assert report.question_block_summary() == "2 of 3 pages"+++def test_a_run_with_no_template_pages_says_so_rather_than_reporting_zero():+ """ "Nothing was measured" and "nothing was found" are different facts.++ Without the denominator, a run that generated only model pages would look+ identical to one where every template lost its questions block.+ """+ report = GenerationReport.from_pages([_page("repo_overview:.", "repo_overview", _WITHOUT)])++ assert report.question_blocks == {"eligible_pages": 0, "with_questions": 0}+ assert report.question_block_summary() == "not measured (0 template pages)"+ assert report.questions_missing is False+++def test_the_check_flags_a_run_where_the_block_stopped_rendering():+ """The failure this exists to catch: eligible pages, no questions on them."""+ report = GenerationReport.from_pages(+ [+ _page("file_page:a.py", "file_page", _WITHOUT),+ _page("file_page:b.py", "file_page", _WITHOUT),+ ]+ )++ assert report.question_blocks == {"eligible_pages": 2, "with_questions": 0}+ assert report.questions_missing is True+++def test_a_full_run_is_not_flagged():+ report = GenerationReport.from_pages([_page("file_page:a.py", "file_page", _WITH)])++ assert report.questions_missing is False+++def test_the_check_prints_even_when_it_measured_nothing():+ """Every check renders every time.++ A hidden zero makes "the check did not run" indistinguishable from "the+ check passed", which is the whole reason the checks table prints rows at+ zero rather than skipping them.+ """+ from io import StringIO++ from rich.console import Console++ from repowise.core.generation.report import render_generation_checks++ buffer = StringIO()+ render_generation_checks(GenerationReport.from_pages([]), Console(file=buffer, width=100))++ assert "Question-shaped text" in buffer.getvalue()diff --git a/tests/unit/generation/test_question_text_reaches_search.py b/tests/unit/generation/test_question_text_reaches_search.pynew file mode 100644index 000000000..4d56d84e3--- /dev/null+++ b/tests/unit/generation/test_question_text_reaches_search.py@@ -0,0 +1,214 @@+"""The questions block has to reach the index, not just the rendered page.++Question-shaped text exists for retrieval. Every query is a question and these+pages are declarative reference text, so the two share almost no vocabulary —+that gap is the whole reason for the block. If the questions render for a+reader but never land in the full-text row, the block is decoration and the+gap it was written to close is still open.++The write path makes this easy to get wrong quietly: ``FullTextSearch.index``+is handed ``page.content`` verbatim, so nothing would raise if the block were+dropped, moved into metadata, or added after the row was written. This asserts+the round trip rather than the render.+"""++from __future__ import annotations++from pathlib import Path++import jinja2+import pytest+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine+from sqlalchemy.pool import StaticPool++from repowise.core.generation.context_assembler import FilePageContext, SymbolSpotlightContext+from repowise.core.generation.page_generator.structural import (+ as_markdown,+ oneline,+ signature,+)+from repowise.core.persistence.crud import upsert_page, upsert_repository+from repowise.core.persistence.database import init_db+from repowise.core.persistence.search import FullTextSearch++QUESTIONS_HEADING = "## Questions this page answers"+++def jinja_env() -> jinja2.Environment:+ templates_dir = (+ Path(__file__).parents[3]+ / "packages"+ / "core"+ / "src"+ / "repowise"+ / "core"+ / "generation"+ / "templates"+ )+ env = jinja2.Environment(+ loader=jinja2.FileSystemLoader(str(templates_dir)),+ undefined=jinja2.StrictUndefined,+ autoescape=False,+ )+ env.filters.setdefault("oneline", oneline)+ env.filters.setdefault("as_markdown", as_markdown)+ env.filters.setdefault("signature", signature)+ return env+++async def fts():+ engine = create_async_engine(+ "sqlite+aiosqlite:///:memory:",+ connect_args={"check_same_thread": False},+ poolclass=StaticPool,+ )+ await init_db(engine)+ search = FullTextSearch(engine)+ await search.ensure_index()+ yield engine, search+ await engine.dispose()+++def _file_page(**overrides) -> FilePageContext:+ base = dict(+ file_path="python_pkg/traverser.py",+ language="python",+ docstring="Walks the tree.",+ symbols=[+ {+ "name": "FileTraverser",+ "kind": "class",+ "signature": "class FileTraverser:",+ "docstring": "Traverse.",+ "visibility": "public",+ "is_async": False,+ "complexity_estimate": 1,+ "decorators": [],+ "parent_name": None,+ "start_line": 1,+ "end_line": 10,+ }+ ],+ imports=[],+ exports=["FileTraverser"],+ file_source_snippet="class FileTraverser: pass",+ pagerank_score=0.5,+ betweenness_score=0.1,+ community_id=0,+ dependents=["main.py"],+ dependencies=["python_pkg/models.py"],+ is_api_contract=False,+ is_entry_point=False,+ is_test=False,+ parse_errors=[],+ estimated_tokens=50,+ )+ base.update(overrides)+ return FilePageContext(**base)+++async def _index(engine, search, *, page_id, page_type, title, target_path, content):+ """Write the page the way the product writes it: SQL row plus FTS row.++ Both halves matter — ``search`` joins ``wiki_pages`` for the page type and+ target path, so a row present in only one of the two is not findable.+ """+ factory = async_sessionmaker(engine, expire_on_commit=False)+ async with factory() as session:+ repo = await upsert_repository(session, name="r", local_path="/tmp/r")+ await session.commit()+ await upsert_page(+ session,+ page_id=page_id,+ repository_id=repo.id,+ page_type=page_type,+ title=title,+ content=content,+ summary="",+ target_path=target_path,+ source_hash="h",+ model_name="template",+ provider_name="template",+ )+ await session.commit()+ await search.index(page_id, title, content, summary="", target_path=target_path)+++async def _matches(search: FullTextSearch, query: str) -> list[str]:+ results = await search.search(query, limit=10)+ return [r.page_id for r in results]+++async def test_a_file_pages_questions_are_searchable(jinja_env, fts):+ """The round trip: render, index, then find the page by its question."""+ engine, search = fts+ content = jinja_env.get_template("file_page.j2").render(ctx=_file_page())+ assert QUESTIONS_HEADING in content++ page_id = "file_page:python_pkg/traverser.py"+ await _index(+ engine,+ search,+ page_id=page_id,+ page_type="file_page",+ title="File: python_pkg/traverser.py",+ target_path="python_pkg/traverser.py",+ content=content,+ )++ # The identifier is what makes the question worth indexing at all.+ assert page_id in await _matches(search, "FileTraverser")+ # And the question shape itself is in the row, not only the reference text.+ assert page_id in await _matches(search, "Questions")+++async def test_a_spotlights_questions_are_searchable(jinja_env, fts):+ ctx = SymbolSpotlightContext(+ symbol_name="FileTraverser",+ qualified_name="python_pkg.traverser.FileTraverser",+ kind="class",+ signature="class FileTraverser:",+ docstring="Traverse.",+ file_path="python_pkg/traverser.py",+ decorators=[],+ is_async=False,+ complexity_estimate=1,+ callers=["main.py"],+ )+ engine, search = fts+ content = jinja_env.get_template("symbol_spotlight.j2").render(ctx=ctx)+ assert QUESTIONS_HEADING in content++ page_id = "symbol_spotlight:python_pkg/traverser.py::FileTraverser"+ await _index(+ engine,+ search,+ page_id=page_id,+ page_type="symbol_spotlight",+ title="Symbol: python_pkg.traverser.FileTraverser",+ target_path="python_pkg/traverser.py::FileTraverser",+ content=content,+ )++ assert page_id in await _matches(search, "FileTraverser")+ assert page_id in await _matches(search, "Questions")+++async def test_the_block_is_not_what_the_page_summary_reads(jinja_env):+ """Placement, asserted where it is load-bearing.++ ``_extract_summary`` takes the first prose paragraph, and that summary is+ what search results, the wiki list and ``get_context`` display. A questions+ block near the top of the page would become that summary and every file+ page in the wiki would introduce itself with a list of questions.+ """+ from repowise.core.generation.page_generator.helpers import _extract_summary++ content = jinja_env.get_template("file_page.j2").render(ctx=_file_page())+ summary = _extract_summary(content)++ assert summary+ assert "Questions this page answers" not in summary+ assert content.index(QUESTIONS_HEADING) > content.index("## Overview")diff --git a/tests/unit/generation/test_templates.py b/tests/unit/generation/test_templates.pyindex 2865c0fc4..405c3ea40 100644--- a/tests/unit/generation/test_templates.py+++ b/tests/unit/generation/test_templates.py@@ -197,6 +197,73 @@ def test_file_page_always_says_what_the_file_is(jinja_env, bare_file_page_ctx):assert "_No " not in result+# ---------------------------------------------------------------------------+# Question-shaped text+#+# Every query is a question and almost every page is a declarative reference,+# so the two share close to no vocabulary. Module pages already close that gap;+# these two page types are the bulk of the corpus and had none of it.+# ---------------------------------------------------------------------------++QUESTIONS_HEADING = "## Questions this page answers"+++def _questions_block(rendered: str) -> str:+ """Just the questions, without the footer — which also says "import".++ Asserts the heading starts a line of its own on the way past. Jinja's+ whitespace control makes it easy to glue it onto the end of the previous+ section, and every other assertion here still passes when that happens:+ splitting on the heading finds it wherever it sits, and gluing produces+ *fewer* blank lines, not more, so the no-gap check passes too. The only+ thing that catches it is looking at the rendered markdown.+ """+ assert QUESTIONS_HEADING in rendered, "no questions block was rendered"+ assert f"\n{QUESTIONS_HEADING}" in rendered, "the heading is glued to the previous line"+ return rendered.split(QUESTIONS_HEADING, 1)[1].split("---", 1)[0]+++def test_file_page_questions_carry_the_real_identifiers(jinja_env, file_page_ctx):+ """A templated question is only worth anything if it names this file.++ Thousands of pages asking the same question differ only in the identifier+ they carry, so the identifier is the entire signal. "What does this file+ export?" would be one sentence repeated across the corpus.+ """+ block = _questions_block(render(jinja_env, "file_page.j2", file_page_ctx))+ assert "`python_pkg/calculator.py`" in block+ assert "`Calculator`" in block+ assert block.count("?") >= 2+++def test_file_page_asks_at_most_three_questions(jinja_env, file_page_ctx):+ """A fully-connected file qualifies for more than are worth carrying.++ Past three the block stops being question-shaped text and becomes padding+ on every page in the wiki.+ """+ block = _questions_block(render(jinja_env, "file_page.j2", file_page_ctx))+ assert block.count("\n- ") <= 3+++def test_file_page_asks_only_what_its_structure_can_answer(jinja_env, bare_file_page_ctx):+ """A question whose answer is not on the page is worse than no question.++ It puts the words of a promise into the index and then breaks it, which is+ the retrieval failure this block exists to close.+ """+ rendered = render(jinja_env, "file_page.j2", bare_file_page_ctx)+ assert QUESTIONS_HEADING not in rendered+++def test_file_page_asks_about_importers_only_when_it_has_them(+ jinja_env, file_page_ctx, bare_file_page_ctx+):+ with_edges = _questions_block(render(jinja_env, "file_page.j2", file_page_ctx))+ assert "What imports" in with_edges+ assert "What imports" not in render(jinja_env, "file_page.j2", bare_file_page_ctx)++# ---------------------------------------------------------------------------# module_page.j2# ---------------------------------------------------------------------------@@ -390,6 +457,33 @@ def test_spotlight_leaves_no_gap_where_a_section_was_dropped(assert not result.startswith("\n")+def test_spotlight_questions_carry_the_symbol_and_its_file(jinja_env, symbol_spotlight_ctx):+ block = _questions_block(render(jinja_env, "symbol_spotlight.j2", symbol_spotlight_ctx))+ assert "`add`" in block+ assert "python_pkg.calculator.add" in block+++def test_spotlight_asks_about_importers_only_when_it_has_them(+ jinja_env, symbol_spotlight_ctx, unimported_spotlight_ctx+):+ """The importer question is the only one this page type can condition on.++ A spotlight's context carries no layer, no dependents and no dependencies,+ and ``callers`` is files importing the defining module rather than verified+ call sites — so "what calls this?" is a question the page cannot honestly+ ask, let alone answer. Two questions plus a conditional third is the whole+ honest set, and padding it would mean asking one it answers falsely.+ """+ with_callers = _questions_block(render(jinja_env, "symbol_spotlight.j2", symbol_spotlight_ctx))+ assert "Which files import" in with_callers++ without = _questions_block(render(jinja_env, "symbol_spotlight.j2", unimported_spotlight_ctx))+ assert "Which files import" not in without+ # The two it can always ask survive on a symbol with no edges at all.+ assert "`_normalise`" in without+ assert without.count("?") >= 2++# ---------------------------------------------------------------------------# architecture_diagram.j2# ---------------------------------------------------------------------------