Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(generation): a symbol spotlight carries a render key, and update sweeps it (#1208)
Swati Ahuja40d71b4ec0b8
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.pyindex 501d94256..35ee677f7 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py@@ -435,24 +435,34 @@ def _head_commit_ts(repo_path) -> float | None:def _current_renderer_fingerprint(repo_path) -> str:- """This release's fingerprint for the file-page renderer, or '' on failure.+ """This release's fingerprint for the structural renderers, or '' on failure.- Cheap: reads one template and a few constants, no parse. An empty result+ Covers every template with no model path behind it, because this value is+ what decides whether a repository with no new commits is "already up to+ date". Fingerprinting only the file page meant a release that improved the+ spotlight template alone returned before doing anything, and the change+ never reached an existing wiki.++ Cheap: reads the templates and a few constants, no parse. An empty resultdisables the renderer-staleness gate for this run rather than risking aspurious full re-render, which is the safe direction."""try:from repowise.core.generation.page_generator.structural import (FILE_PAGE_TEMPLATE,+ SYMBOL_SPOTLIGHT_TEMPLATE,structural_fingerprint,)language, style_fp, template_dir = _renderer_inputs(repo_path)- return structural_fingerprint(- FILE_PAGE_TEMPLATE,- language=language,- style_fingerprint=style_fp,- template_dir=template_dir,+ return ":".join(+ structural_fingerprint(+ template,+ language=language,+ style_fingerprint=style_fp,+ template_dir=template_dir,+ )+ for template in (FILE_PAGE_TEMPLATE, SYMBOL_SPOTLIGHT_TEMPLATE))except Exception as exc:log.debug("update.renderer_fingerprint_failed", error=str(exc))@@ -460,7 +470,11 @@ def _current_renderer_fingerprint(repo_path) -> str:def _stale_renderer_paths(repo_path, parsed_files) -> list[str]:- """File paths whose stored page predates the current structural renderer.+ """File paths whose stored pages predate the current structural renderers.++ Covers file pages and symbol spotlights. Both sweeps return file paths, so+ the union is the set of files to re-parse; regenerating a file re-renders+ every page derived from it, including its spotlights.Never raises: a failure here means the run behaves exactly as it didbefore the fingerprint existed, which is the safe direction. The style and@@ -468,21 +482,29 @@ def _stale_renderer_paths(repo_path, parsed_files) -> list[str]:would look stale on every run."""try:- from repowise.core.generation.page_generator.structural import stale_file_page_paths+ from repowise.core.generation.page_generator.structural import (+ stale_file_page_paths,+ stale_spotlight_paths,+ )- from .deterministic import load_file_page_render_keys+ from .deterministic import load_file_page_render_keys, load_spotlight_render_keys- stored = load_file_page_render_keys(repo_path)- if not stored:- return []+ parsed_files = list(parsed_files)language, style_fp, template_dir = _renderer_inputs(repo_path)- return stale_file_page_paths(- stored,- parsed_files,- language=language,- style_fingerprint=style_fp,- template_dir=template_dir,- )+ kwargs = {+ "language": language,+ "style_fingerprint": style_fp,+ "template_dir": template_dir,+ }++ stale: list[str] = []+ stored_pages = load_file_page_render_keys(repo_path)+ if stored_pages:+ stale.extend(stale_file_page_paths(stored_pages, parsed_files, **kwargs))+ stored_spotlights = load_spotlight_render_keys(repo_path)+ if stored_spotlights:+ stale.extend(stale_spotlight_paths(stored_spotlights, parsed_files, **kwargs))+ return list(dict.fromkeys(stale))except Exception as exc:log.debug("update.stale_renderer_check_failed", error=str(exc))return []@@ -1711,7 +1733,5 @@ def run_update(elapsed=elapsed,degraded=degraded,)- _render_update_report(- generated_pages, affected, new_decision_markers, elapsed, detail=verbose- )+ _render_update_report(generated_pages, affected, new_decision_markers, elapsed, detail=verbose)return UpdateOutcome.REGENERATEDdiff --git a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.pyindex 39f4d651c..1d702df02 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py@@ -365,3 +365,61 @@ async def _load_file_page_render_keys(repo_path: Path) -> dict[str, str]:return {}finally:await engine.dispose()+++def load_spotlight_render_keys(repo_path: Path) -> dict[str, list[str]]:+ """``defining file path -> render keys`` for the wiki's symbol spotlights.++ Grouped by file rather than returned per page id because that is the unit+ the sweep and the regeneration both work in: a spotlight's target path is+ ``<file>::<symbol>``, and one stale spotlight makes its whole file the work+ item. A row with no key is kept as an empty string, which is what a page+ written before spotlights carried a fingerprint looks like.+ """+ return run_async(_load_spotlight_render_keys(repo_path))+++async def _load_spotlight_render_keys(repo_path: Path) -> dict[str, list[str]]:+ from repowise.cli.helpers import get_db_url_for_repo+ from repowise.core.persistence import create_engine, create_session_factory, get_session++ engine = create_engine(get_db_url_for_repo(repo_path))+ try:+ import json++ from sqlalchemy import select as sa_select++ from repowise.core.generation.page_generator.structural import RENDER_KEY+ from repowise.core.persistence.models import Page++ async with get_session(create_session_factory(engine)) as session:+ rows = await session.execute(+ sa_select(Page.target_path, Page.metadata_json).where(+ Page.page_type == "symbol_spotlight"+ )+ )+ keys: dict[str, list[str]] = {}+ for target_path, meta_json in rows:+ if not target_path or "::" not in target_path:+ # Not a shape this sweep can attribute to a file. Skipping+ # it means the page is left alone rather than re-rendered+ # on every run against an expectation we cannot compute.+ continue+ try:+ meta = json.loads(meta_json or "{}")+ except (TypeError, ValueError):+ meta = {}+ file_path = target_path.split("::", 1)[0]+ keys.setdefault(file_path, []).append(str(meta.get(RENDER_KEY) or ""))+ return keys+ except Exception as exc:+ # Same contract as the file-page loader: never block an update on it.+ # Said out loud rather than swallowed, because the failure is silent by+ # construction — returning nothing makes every spotlight look current,+ # so an improved template would quietly never land and the run would+ # still report success. Printed rather than logged: the CLI sets the+ # logger to ERROR, so a warning there would not survive.+ console.print(f"[yellow]Could not read spotlight render keys: {exc}[/yellow]")+ return {}+ finally:+ await engine.dispose()diff --git a/packages/core/src/repowise/core/generation/page_generator/helpers.py b/packages/core/src/repowise/core/generation/page_generator/helpers.pyindex 915a8c7f5..eb9aca98b 100644--- a/packages/core/src/repowise/core/generation/page_generator/helpers.py+++ b/packages/core/src/repowise/core/generation/page_generator/helpers.py@@ -219,9 +219,7 @@ def _select_clone_representatives(# Near-clones usually share a PageRank (often 0.0), so the path breaks# the tie. Without it the survivor of each cluster changes between# runs, and with it which file gets a page at all.- members.sort(- key=lambda p: (-pagerank.get(p.file_info.path, 0.0), p.file_info.path)- )+ members.sort(key=lambda p: (-pagerank.get(p.file_info.path, 0.0), p.file_info.path))for loser in members[1:]:drop.add(loser.file_info.path)return dropdiff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.pyindex 985ec5cb0..99cddd50d 100644--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py@@ -112,7 +112,18 @@ class PerTypeGenerationMixin:source_bytes=(source_map or {}).get(parsed.file_info.path, b""),)target = f"{parsed.file_info.path}::{symbol.name}"- return self._structural_symbol_spotlight(ctx, target, f"Symbol: {symbol.qualified_name}")+ # The subject is the defining file's bytes — the same subject the file+ # page uses. A spotlight renders a symbol out of that file, so when the+ # bytes move the spotlight has to be redone anyway, and there is no+ # finer-grained per-symbol hash stored to compare against. Without a+ # subject the page stores no render key at all and the staleness sweep+ # can never see that an improved template has not reached it.+ return self._structural_symbol_spotlight(+ ctx,+ target,+ f"Symbol: {symbol.qualified_name}",+ subject_hash=parsed.content_hash or "",+ )async def generate_module_page(self,diff --git a/packages/core/src/repowise/core/generation/page_generator/structural.py b/packages/core/src/repowise/core/generation/page_generator/structural.pyindex 19c7fbee8..0c4bf16ed 100644--- a/packages/core/src/repowise/core/generation/page_generator/structural.py+++ b/packages/core/src/repowise/core/generation/page_generator/structural.py@@ -158,6 +158,7 @@ def signature(value: object, limit: int = 120) -> str:_TEMPLATES_DIR = Path(__file__).parent.parent / "templates"FILE_PAGE_TEMPLATE = "file_page.j2"+SYMBOL_SPOTLIGHT_TEMPLATE = "symbol_spotlight.j2"# Metadata key holding a structural page's render fingerprint. It lives in# metadata rather than in a column of its own because ``GeneratedPage`` already@@ -280,6 +281,50 @@ def stale_file_page_paths(return stale+def stale_spotlight_paths(+ stored_keys_by_path: Mapping[str, Iterable[str]],+ parsed_files: Iterable[Any],+ *,+ language: str = "en",+ style_fingerprint: str = "",+ template_dir: Path | None = None,+) -> list[str]:+ """Files whose stored symbol spotlights came from an older renderer.++ The same idea as :func:`stale_file_page_paths` and deliberately a separate+ function, because the two are keyed differently. A file page is one page+ per path, so it can be looked up by page id. A file has many spotlights,+ one per selected symbol, and which symbols were selected on the run that+ wrote them is not reconstructible here — so the caller groups the stored+ keys by defining file and this compares against the set.++ Returns **file paths**, not page ids, because regeneration is driven by+ file: ``update`` re-parses a path and re-renders every page derived from+ it. One stale spotlight therefore makes its defining file the unit of work.+ """+ fingerprint = structural_fingerprint(+ SYMBOL_SPOTLIGHT_TEMPLATE,+ language=language,+ style_fingerprint=style_fingerprint,+ template_dir=template_dir,+ )+ stale: list[str] = []+ for parsed in parsed_files:+ subject_hash = getattr(parsed, "content_hash", "") or ""+ if not subject_hash:+ # Same reasoning as the file-page sweep: no stable subject means no+ # stable expectation, and treating it as stale re-renders forever.+ continue+ stored = stored_keys_by_path.get(parsed.file_info.path)+ if not stored:+ # No spotlight stored for this file is absent, not stale.+ continue+ expected = structural_content_hash(subject_hash, fingerprint)+ if any(key != expected for key in stored):+ stale.append(parsed.file_info.path)+ return stale++class StructuralRenderMixin:"""Template-only renderers, mixed into PageGenerator.@@ -412,7 +457,7 @@ class StructuralRenderMixin:page_type="symbol_spotlight",target_path=target_path,title=title,- template="symbol_spotlight.j2",+ template=SYMBOL_SPOTLIGHT_TEMPLATE,subject_hash=subject_hash,ctx=ctx,)diff --git a/tests/unit/cli/test_update_renderer_gate.py b/tests/unit/cli/test_update_renderer_gate.pynew file mode 100644index 000000000..868b9fe3e--- /dev/null+++ b/tests/unit/cli/test_update_renderer_gate.py@@ -0,0 +1,59 @@+"""The gate that decides a quiet repository is "already up to date".++``update`` returns early when there are no new commits, no config change and no+renderer change. That last term is the only route a template improvement has+into a repository nobody has touched, so whatever it fingerprints is the set of+templates a release can actually ship improvements to. A template left out of+it is a template whose changes never land.+"""++from __future__ import annotations++from pathlib import Path++from repowise.cli.commands.update_cmd.command import _current_renderer_fingerprint+from repowise.core.generation.page_generator.structural import (+ FILE_PAGE_TEMPLATE,+ SYMBOL_SPOTLIGHT_TEMPLATE,+ structural_fingerprint,+)+++def _fingerprint_of(template: str, source: str) -> str:+ return structural_fingerprint(template, source=source)+++def test_the_gate_covers_every_template_with_no_model_path(tmp_path: Path):+ """Both structural templates have to be inside the gate.++ A spotlight is rendered by a template and nothing else — no model will come+ along later and rewrite it — so if its bytes are outside this value, a+ release that improves only the spotlight template reports "already up to+ date" and does nothing.+ """+ current = _current_renderer_fingerprint(tmp_path)++ assert _fingerprint_of(FILE_PAGE_TEMPLATE, _read(FILE_PAGE_TEMPLATE)) in current+ assert _fingerprint_of(SYMBOL_SPOTLIGHT_TEMPLATE, _read(SYMBOL_SPOTLIGHT_TEMPLATE)) in current+++def test_the_gate_is_stable_across_runs_of_one_release(tmp_path: Path):+ """It has to agree with itself, or every update rewrites every page."""+ assert _current_renderer_fingerprint(tmp_path) == _current_renderer_fingerprint(tmp_path)+++def test_the_gate_distinguishes_the_two_templates(tmp_path: Path):+ """Folding both in must not collapse them to one value.++ If editing either template produced the same fingerprint, the gate could+ not tell which pages to sweep and would have to re-render both types.+ """+ assert _fingerprint_of(FILE_PAGE_TEMPLATE, "same bytes") != _fingerprint_of(+ SYMBOL_SPOTLIGHT_TEMPLATE, "same bytes"+ )+++def _read(template: str) -> str:+ from repowise.core.generation.page_generator.structural import _TEMPLATES_DIR++ return (_TEMPLATES_DIR / template).read_text(encoding="utf-8")diff --git a/tests/unit/generation/test_structural_salt.py b/tests/unit/generation/test_structural_salt.pyindex 6cb598ba0..97ab99a37 100644--- a/tests/unit/generation/test_structural_salt.py+++ b/tests/unit/generation/test_structural_salt.py@@ -21,7 +21,9 @@ from repowise.core.generation.models import compute_page_idfrom repowise.core.generation.page_generator.structural import (FILE_PAGE_TEMPLATE,RENDER_KEY,+ SYMBOL_SPOTLIGHT_TEMPLATE,stale_file_page_paths,+ stale_spotlight_paths,structural_content_hash,structural_fingerprint,)@@ -187,6 +189,88 @@ def test_a_style_switch_makes_every_page_stale():assert stale_file_page_paths(stored, parsed, style_fingerprint="caveman") == ["src/a.py"]+# ---------------------------------------------------------------------------+# The same sweep for symbol spotlights+#+# Keyed differently on purpose: a file has one file page and many spotlights,+# and which symbols were selected on the run that wrote them is not+# reconstructible here, so the caller groups stored keys by defining file.+# ---------------------------------------------------------------------------+++def _spotlight_hash(content_hash: str, **kwargs) -> str:+ return structural_content_hash(+ content_hash, structural_fingerprint(SYMBOL_SPOTLIGHT_TEMPLATE, **kwargs)+ )+++def test_no_spotlight_is_stale_when_the_renderer_has_not_moved():+ parsed = [_parsed("src/a.py"), _parsed("src/b.py", "other-bytes")]+ stored = {+ "src/a.py": [_spotlight_hash("hash-of-bytes"), _spotlight_hash("hash-of-bytes")],+ "src/b.py": [_spotlight_hash("other-bytes")],+ }+ assert stale_spotlight_paths(stored, parsed) == []+++def test_a_spotlight_template_change_makes_its_file_stale():+ """The case this exists for: the template moved, so the page has to be redone."""+ parsed = [_parsed("src/a.py")]+ old = structural_fingerprint(SYMBOL_SPOTLIGHT_TEMPLATE, source="# an older template")+ stored = {"src/a.py": [structural_content_hash("hash-of-bytes", old)]}+ assert stale_spotlight_paths(stored, parsed) == ["src/a.py"]+++def test_one_stale_spotlight_makes_the_whole_file_the_unit_of_work():+ """Regeneration is per file, so a single mismatch is enough to select it."""+ parsed = [_parsed("src/a.py")]+ stored = {"src/a.py": [_spotlight_hash("hash-of-bytes"), "written-by-something-older"]}+ assert stale_spotlight_paths(stored, parsed) == ["src/a.py"]+++def test_spotlights_predating_the_salt_are_refreshed_once():+ """Every spotlight in every existing wiki stores no key at all.++ That is not a corner case: spotlights were rendered without a subject hash,+ so nothing was ever stamped. This is the whole population on the first run+ that carries the fix.+ """+ parsed = [_parsed("src/a.py")]+ assert stale_spotlight_paths({"src/a.py": [""]}, parsed) == ["src/a.py"]+++def test_a_file_with_no_spotlights_is_not_stale():+ """Absent is not stale — most files never get a spotlight at all."""+ parsed = [_parsed("src/no_symbols.py")]+ assert stale_spotlight_paths({}, parsed) == []+ assert stale_spotlight_paths({"src/no_symbols.py": []}, parsed) == []+++def test_a_file_with_no_content_hash_is_skipped_by_the_spotlight_sweep():+ parsed = [_parsed("src/unparsed.py", content_hash="")]+ assert stale_spotlight_paths({"src/unparsed.py": ["whatever"]}, parsed) == []+++def test_the_two_sweeps_do_not_share_a_fingerprint():+ """Each template is fingerprinted separately, or editing one would falsely+ mark the other's pages stale and re-render the whole wiki."""+ assert structural_fingerprint(FILE_PAGE_TEMPLATE) != structural_fingerprint(+ SYMBOL_SPOTLIGHT_TEMPLATE+ )+ # A store written by the current release is stale under neither sweep.+ parsed = [_parsed("src/a.py")]+ assert (+ stale_file_page_paths(_stored({"src/a.py": _current_hash("hash-of-bytes")}), parsed) == []+ )+ assert stale_spotlight_paths({"src/a.py": [_spotlight_hash("hash-of-bytes")]}, parsed) == []+++def test_a_style_switch_makes_every_spotlight_stale():+ parsed = [_parsed("src/a.py")]+ stored = {"src/a.py": [_spotlight_hash("hash-of-bytes")]}+ assert stale_spotlight_paths(stored, parsed, style_fingerprint="caveman") == ["src/a.py"]++def test_a_custom_style_template_override_is_what_gets_fingerprinted(tmp_path):"""A style shipping its own file_page.j2 must fingerprint that, not the base.diff --git a/tests/unit/generation/test_styles.py b/tests/unit/generation/test_styles.pyindex 30a847201..7c6bc8076 100644--- a/tests/unit/generation/test_styles.py+++ b/tests/unit/generation/test_styles.py@@ -252,6 +252,56 @@ async def test_style_change_moves_a_file_page_reuse_hash(assert base_page.metadata["render_key"] != other_page.metadata["render_key"]+async def test_a_symbol_spotlight_stores_a_render_key(+ sample_parsed_file, sample_graph, graph_metrics, sample_source_bytes+):+ """A spotlight has no model path either, so it needs the same salt.++ Without a stored key there is nothing for a later run to disagree with, so+ an improved spotlight template can never be shown not to have landed, and+ the page keeps whatever it said when it was first written.+ """+ _, gen = _make_gen(GenerationConfig(), sample_source_bytes)+ page = await gen.generate_symbol_spotlight(+ sample_parsed_file.symbols[0],+ sample_parsed_file,+ graph_metrics["pagerank"],+ sample_graph,+ {sample_parsed_file.file_info.path: sample_source_bytes},+ )++ assert page.metadata.get("render_key")+++async def test_a_spotlight_render_key_moves_when_its_template_does(+ sample_parsed_file, sample_graph, graph_metrics, sample_source_bytes+):+ """The key is only worth storing if a template edit actually moves it."""+ from repowise.core.generation.page_generator.structural import (+ SYMBOL_SPOTLIGHT_TEMPLATE,+ structural_content_hash,+ structural_fingerprint,+ )++ _, gen = _make_gen(GenerationConfig(), sample_source_bytes)+ page = await gen.generate_symbol_spotlight(+ sample_parsed_file.symbols[0],+ sample_parsed_file,+ graph_metrics["pagerank"],+ sample_graph,+ {sample_parsed_file.file_info.path: sample_source_bytes},+ )++ subject = sample_parsed_file.content_hash+ current = structural_content_hash(subject, structural_fingerprint(SYMBOL_SPOTLIGHT_TEMPLATE))+ older = structural_content_hash(+ subject, structural_fingerprint(SYMBOL_SPOTLIGHT_TEMPLATE, source="# an older template")+ )++ assert page.metadata["render_key"] == current+ assert page.metadata["render_key"] != older++async def test_onboarding_page_type_constant_matches_system_prompts():"""ONBOARDING_PAGE_TYPE must match the key used by the onboarding generators."""from repowise.core.generation.page_generator import SYSTEM_PROMPTS