Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(generation): a symbol spotlight carries a render key, and update sweeps it (#1208)

Swati Ahuja40d71b4ec0b8
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
index 501d94256..35ee677f7 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
@@ -435,24 +435,34 @@ def _head_commit_ts(repo_path) -> float | None:
def _current_renderer_fingerprint(repo_path) -> str:
- """This release's fingerprint for the file-page renderer, or '' on failure.
+ """This release's fingerprint for the structural renderers, or '' on failure.
- Cheap: reads one template and a few constants, no parse. An empty result
+ Covers every template with no model path behind it, because this value is
+ what decides whether a repository with no new commits is "already up to
+ date". Fingerprinting only the file page meant a release that improved the
+ spotlight template alone returned before doing anything, and the change
+ never reached an existing wiki.
+
+ Cheap: reads the templates and a few constants, no parse. An empty result
disables the renderer-staleness gate for this run rather than risking a
spurious full re-render, which is the safe direction.
"""
try:
from repowise.core.generation.page_generator.structural import (
FILE_PAGE_TEMPLATE,
+ SYMBOL_SPOTLIGHT_TEMPLATE,
structural_fingerprint,
)
language, style_fp, template_dir = _renderer_inputs(repo_path)
- return structural_fingerprint(
- FILE_PAGE_TEMPLATE,
- language=language,
- style_fingerprint=style_fp,
- template_dir=template_dir,
+ return ":".join(
+ structural_fingerprint(
+ template,
+ language=language,
+ style_fingerprint=style_fp,
+ template_dir=template_dir,
+ )
+ for template in (FILE_PAGE_TEMPLATE, SYMBOL_SPOTLIGHT_TEMPLATE)
)
except Exception as exc:
log.debug("update.renderer_fingerprint_failed", error=str(exc))
@@ -460,7 +470,11 @@ def _current_renderer_fingerprint(repo_path) -> str:
def _stale_renderer_paths(repo_path, parsed_files) -> list[str]:
- """File paths whose stored page predates the current structural renderer.
+ """File paths whose stored pages predate the current structural renderers.
+
+ Covers file pages and symbol spotlights. Both sweeps return file paths, so
+ the union is the set of files to re-parse; regenerating a file re-renders
+ every page derived from it, including its spotlights.
Never raises: a failure here means the run behaves exactly as it did
before the fingerprint existed, which is the safe direction. The style and
@@ -468,21 +482,29 @@ def _stale_renderer_paths(repo_path, parsed_files) -> list[str]:
would look stale on every run.
"""
try:
- from repowise.core.generation.page_generator.structural import stale_file_page_paths
+ from repowise.core.generation.page_generator.structural import (
+ stale_file_page_paths,
+ stale_spotlight_paths,
+ )
- from .deterministic import load_file_page_render_keys
+ from .deterministic import load_file_page_render_keys, load_spotlight_render_keys
- stored = load_file_page_render_keys(repo_path)
- if not stored:
- return []
+ parsed_files = list(parsed_files)
language, style_fp, template_dir = _renderer_inputs(repo_path)
- return stale_file_page_paths(
- stored,
- parsed_files,
- language=language,
- style_fingerprint=style_fp,
- template_dir=template_dir,
- )
+ kwargs = {
+ "language": language,
+ "style_fingerprint": style_fp,
+ "template_dir": template_dir,
+ }
+
+ stale: list[str] = []
+ stored_pages = load_file_page_render_keys(repo_path)
+ if stored_pages:
+ stale.extend(stale_file_page_paths(stored_pages, parsed_files, **kwargs))
+ stored_spotlights = load_spotlight_render_keys(repo_path)
+ if stored_spotlights:
+ stale.extend(stale_spotlight_paths(stored_spotlights, parsed_files, **kwargs))
+ return list(dict.fromkeys(stale))
except Exception as exc:
log.debug("update.stale_renderer_check_failed", error=str(exc))
return []
@@ -1711,7 +1733,5 @@ def run_update(
elapsed=elapsed,
degraded=degraded,
)
- _render_update_report(
- generated_pages, affected, new_decision_markers, elapsed, detail=verbose
- )
+ _render_update_report(generated_pages, affected, new_decision_markers, elapsed, detail=verbose)
return UpdateOutcome.REGENERATED
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
index 39f4d651c..1d702df02 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
@@ -365,3 +365,61 @@ async def _load_file_page_render_keys(repo_path: Path) -> dict[str, str]:
return {}
finally:
await engine.dispose()
+
+
+def load_spotlight_render_keys(repo_path: Path) -> dict[str, list[str]]:
+ """``defining file path -> render keys`` for the wiki's symbol spotlights.
+
+ Grouped by file rather than returned per page id because that is the unit
+ the sweep and the regeneration both work in: a spotlight's target path is
+ ``<file>::<symbol>``, and one stale spotlight makes its whole file the work
+ item. A row with no key is kept as an empty string, which is what a page
+ written before spotlights carried a fingerprint looks like.
+ """
+ return run_async(_load_spotlight_render_keys(repo_path))
+
+
+async def _load_spotlight_render_keys(repo_path: Path) -> dict[str, list[str]]:
+ from repowise.cli.helpers import get_db_url_for_repo
+ from repowise.core.persistence import create_engine, create_session_factory, get_session
+
+ engine = create_engine(get_db_url_for_repo(repo_path))
+ try:
+ import json
+
+ from sqlalchemy import select as sa_select
+
+ from repowise.core.generation.page_generator.structural import RENDER_KEY
+ from repowise.core.persistence.models import Page
+
+ async with get_session(create_session_factory(engine)) as session:
+ rows = await session.execute(
+ sa_select(Page.target_path, Page.metadata_json).where(
+ Page.page_type == "symbol_spotlight"
+ )
+ )
+ keys: dict[str, list[str]] = {}
+ for target_path, meta_json in rows:
+ if not target_path or "::" not in target_path:
+ # Not a shape this sweep can attribute to a file. Skipping
+ # it means the page is left alone rather than re-rendered
+ # on every run against an expectation we cannot compute.
+ continue
+ try:
+ meta = json.loads(meta_json or "{}")
+ except (TypeError, ValueError):
+ meta = {}
+ file_path = target_path.split("::", 1)[0]
+ keys.setdefault(file_path, []).append(str(meta.get(RENDER_KEY) or ""))
+ return keys
+ except Exception as exc:
+ # Same contract as the file-page loader: never block an update on it.
+ # Said out loud rather than swallowed, because the failure is silent by
+ # construction — returning nothing makes every spotlight look current,
+ # so an improved template would quietly never land and the run would
+ # still report success. Printed rather than logged: the CLI sets the
+ # logger to ERROR, so a warning there would not survive.
+ console.print(f"[yellow]Could not read spotlight render keys: {exc}[/yellow]")
+ return {}
+ finally:
+ await engine.dispose()
diff --git a/packages/core/src/repowise/core/generation/page_generator/helpers.py b/packages/core/src/repowise/core/generation/page_generator/helpers.py
index 915a8c7f5..eb9aca98b 100644
--- a/packages/core/src/repowise/core/generation/page_generator/helpers.py
+++ b/packages/core/src/repowise/core/generation/page_generator/helpers.py
@@ -219,9 +219,7 @@ def _select_clone_representatives(
# Near-clones usually share a PageRank (often 0.0), so the path breaks
# the tie. Without it the survivor of each cluster changes between
# runs, and with it which file gets a page at all.
- members.sort(
- key=lambda p: (-pagerank.get(p.file_info.path, 0.0), p.file_info.path)
- )
+ members.sort(key=lambda p: (-pagerank.get(p.file_info.path, 0.0), p.file_info.path))
for loser in members[1:]:
drop.add(loser.file_info.path)
return drop
diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.py
index 985ec5cb0..99cddd50d 100644
--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py
+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py
@@ -112,7 +112,18 @@ class PerTypeGenerationMixin:
source_bytes=(source_map or {}).get(parsed.file_info.path, b""),
)
target = f"{parsed.file_info.path}::{symbol.name}"
- return self._structural_symbol_spotlight(ctx, target, f"Symbol: {symbol.qualified_name}")
+ # The subject is the defining file's bytes — the same subject the file
+ # page uses. A spotlight renders a symbol out of that file, so when the
+ # bytes move the spotlight has to be redone anyway, and there is no
+ # finer-grained per-symbol hash stored to compare against. Without a
+ # subject the page stores no render key at all and the staleness sweep
+ # can never see that an improved template has not reached it.
+ return self._structural_symbol_spotlight(
+ ctx,
+ target,
+ f"Symbol: {symbol.qualified_name}",
+ subject_hash=parsed.content_hash or "",
+ )
async def generate_module_page(
self,
diff --git a/packages/core/src/repowise/core/generation/page_generator/structural.py b/packages/core/src/repowise/core/generation/page_generator/structural.py
index 19c7fbee8..0c4bf16ed 100644
--- a/packages/core/src/repowise/core/generation/page_generator/structural.py
+++ b/packages/core/src/repowise/core/generation/page_generator/structural.py
@@ -158,6 +158,7 @@ def signature(value: object, limit: int = 120) -> str:
_TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
FILE_PAGE_TEMPLATE = "file_page.j2"
+SYMBOL_SPOTLIGHT_TEMPLATE = "symbol_spotlight.j2"
# Metadata key holding a structural page's render fingerprint. It lives in
# metadata rather than in a column of its own because ``GeneratedPage`` already
@@ -280,6 +281,50 @@ def stale_file_page_paths(
return stale
+def stale_spotlight_paths(
+ stored_keys_by_path: Mapping[str, Iterable[str]],
+ parsed_files: Iterable[Any],
+ *,
+ language: str = "en",
+ style_fingerprint: str = "",
+ template_dir: Path | None = None,
+) -> list[str]:
+ """Files whose stored symbol spotlights came from an older renderer.
+
+ The same idea as :func:`stale_file_page_paths` and deliberately a separate
+ function, because the two are keyed differently. A file page is one page
+ per path, so it can be looked up by page id. A file has many spotlights,
+ one per selected symbol, and which symbols were selected on the run that
+ wrote them is not reconstructible here — so the caller groups the stored
+ keys by defining file and this compares against the set.
+
+ Returns **file paths**, not page ids, because regeneration is driven by
+ file: ``update`` re-parses a path and re-renders every page derived from
+ it. One stale spotlight therefore makes its defining file the unit of work.
+ """
+ fingerprint = structural_fingerprint(
+ SYMBOL_SPOTLIGHT_TEMPLATE,
+ language=language,
+ style_fingerprint=style_fingerprint,
+ template_dir=template_dir,
+ )
+ stale: list[str] = []
+ for parsed in parsed_files:
+ subject_hash = getattr(parsed, "content_hash", "") or ""
+ if not subject_hash:
+ # Same reasoning as the file-page sweep: no stable subject means no
+ # stable expectation, and treating it as stale re-renders forever.
+ continue
+ stored = stored_keys_by_path.get(parsed.file_info.path)
+ if not stored:
+ # No spotlight stored for this file is absent, not stale.
+ continue
+ expected = structural_content_hash(subject_hash, fingerprint)
+ if any(key != expected for key in stored):
+ stale.append(parsed.file_info.path)
+ return stale
+
+
class StructuralRenderMixin:
"""Template-only renderers, mixed into PageGenerator.
@@ -412,7 +457,7 @@ class StructuralRenderMixin:
page_type="symbol_spotlight",
target_path=target_path,
title=title,
- template="symbol_spotlight.j2",
+ template=SYMBOL_SPOTLIGHT_TEMPLATE,
subject_hash=subject_hash,
ctx=ctx,
)
diff --git a/tests/unit/cli/test_update_renderer_gate.py b/tests/unit/cli/test_update_renderer_gate.py
new file mode 100644
index 000000000..868b9fe3e
--- /dev/null
+++ b/tests/unit/cli/test_update_renderer_gate.py
@@ -0,0 +1,59 @@
+"""The gate that decides a quiet repository is "already up to date".
+
+``update`` returns early when there are no new commits, no config change and no
+renderer change. That last term is the only route a template improvement has
+into a repository nobody has touched, so whatever it fingerprints is the set of
+templates a release can actually ship improvements to. A template left out of
+it is a template whose changes never land.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from repowise.cli.commands.update_cmd.command import _current_renderer_fingerprint
+from repowise.core.generation.page_generator.structural import (
+ FILE_PAGE_TEMPLATE,
+ SYMBOL_SPOTLIGHT_TEMPLATE,
+ structural_fingerprint,
+)
+
+
+def _fingerprint_of(template: str, source: str) -> str:
+ return structural_fingerprint(template, source=source)
+
+
+def test_the_gate_covers_every_template_with_no_model_path(tmp_path: Path):
+ """Both structural templates have to be inside the gate.
+
+ A spotlight is rendered by a template and nothing else — no model will come
+ along later and rewrite it — so if its bytes are outside this value, a
+ release that improves only the spotlight template reports "already up to
+ date" and does nothing.
+ """
+ current = _current_renderer_fingerprint(tmp_path)
+
+ assert _fingerprint_of(FILE_PAGE_TEMPLATE, _read(FILE_PAGE_TEMPLATE)) in current
+ assert _fingerprint_of(SYMBOL_SPOTLIGHT_TEMPLATE, _read(SYMBOL_SPOTLIGHT_TEMPLATE)) in current
+
+
+def test_the_gate_is_stable_across_runs_of_one_release(tmp_path: Path):
+ """It has to agree with itself, or every update rewrites every page."""
+ assert _current_renderer_fingerprint(tmp_path) == _current_renderer_fingerprint(tmp_path)
+
+
+def test_the_gate_distinguishes_the_two_templates(tmp_path: Path):
+ """Folding both in must not collapse them to one value.
+
+ If editing either template produced the same fingerprint, the gate could
+ not tell which pages to sweep and would have to re-render both types.
+ """
+ assert _fingerprint_of(FILE_PAGE_TEMPLATE, "same bytes") != _fingerprint_of(
+ SYMBOL_SPOTLIGHT_TEMPLATE, "same bytes"
+ )
+
+
+def _read(template: str) -> str:
+ from repowise.core.generation.page_generator.structural import _TEMPLATES_DIR
+
+ return (_TEMPLATES_DIR / template).read_text(encoding="utf-8")
diff --git a/tests/unit/generation/test_structural_salt.py b/tests/unit/generation/test_structural_salt.py
index 6cb598ba0..97ab99a37 100644
--- a/tests/unit/generation/test_structural_salt.py
+++ b/tests/unit/generation/test_structural_salt.py
@@ -21,7 +21,9 @@ from repowise.core.generation.models import compute_page_id
from repowise.core.generation.page_generator.structural import (
FILE_PAGE_TEMPLATE,
RENDER_KEY,
+ SYMBOL_SPOTLIGHT_TEMPLATE,
stale_file_page_paths,
+ stale_spotlight_paths,
structural_content_hash,
structural_fingerprint,
)
@@ -187,6 +189,88 @@ def test_a_style_switch_makes_every_page_stale():
assert stale_file_page_paths(stored, parsed, style_fingerprint="caveman") == ["src/a.py"]
+# ---------------------------------------------------------------------------
+# The same sweep for symbol spotlights
+#
+# Keyed differently on purpose: a file has one file page and many spotlights,
+# and which symbols were selected on the run that wrote them is not
+# reconstructible here, so the caller groups stored keys by defining file.
+# ---------------------------------------------------------------------------
+
+
+def _spotlight_hash(content_hash: str, **kwargs) -> str:
+ return structural_content_hash(
+ content_hash, structural_fingerprint(SYMBOL_SPOTLIGHT_TEMPLATE, **kwargs)
+ )
+
+
+def test_no_spotlight_is_stale_when_the_renderer_has_not_moved():
+ parsed = [_parsed("src/a.py"), _parsed("src/b.py", "other-bytes")]
+ stored = {
+ "src/a.py": [_spotlight_hash("hash-of-bytes"), _spotlight_hash("hash-of-bytes")],
+ "src/b.py": [_spotlight_hash("other-bytes")],
+ }
+ assert stale_spotlight_paths(stored, parsed) == []
+
+
+def test_a_spotlight_template_change_makes_its_file_stale():
+ """The case this exists for: the template moved, so the page has to be redone."""
+ parsed = [_parsed("src/a.py")]
+ old = structural_fingerprint(SYMBOL_SPOTLIGHT_TEMPLATE, source="# an older template")
+ stored = {"src/a.py": [structural_content_hash("hash-of-bytes", old)]}
+ assert stale_spotlight_paths(stored, parsed) == ["src/a.py"]
+
+
+def test_one_stale_spotlight_makes_the_whole_file_the_unit_of_work():
+ """Regeneration is per file, so a single mismatch is enough to select it."""
+ parsed = [_parsed("src/a.py")]
+ stored = {"src/a.py": [_spotlight_hash("hash-of-bytes"), "written-by-something-older"]}
+ assert stale_spotlight_paths(stored, parsed) == ["src/a.py"]
+
+
+def test_spotlights_predating_the_salt_are_refreshed_once():
+ """Every spotlight in every existing wiki stores no key at all.
+
+ That is not a corner case: spotlights were rendered without a subject hash,
+ so nothing was ever stamped. This is the whole population on the first run
+ that carries the fix.
+ """
+ parsed = [_parsed("src/a.py")]
+ assert stale_spotlight_paths({"src/a.py": [""]}, parsed) == ["src/a.py"]
+
+
+def test_a_file_with_no_spotlights_is_not_stale():
+ """Absent is not stale — most files never get a spotlight at all."""
+ parsed = [_parsed("src/no_symbols.py")]
+ assert stale_spotlight_paths({}, parsed) == []
+ assert stale_spotlight_paths({"src/no_symbols.py": []}, parsed) == []
+
+
+def test_a_file_with_no_content_hash_is_skipped_by_the_spotlight_sweep():
+ parsed = [_parsed("src/unparsed.py", content_hash="")]
+ assert stale_spotlight_paths({"src/unparsed.py": ["whatever"]}, parsed) == []
+
+
+def test_the_two_sweeps_do_not_share_a_fingerprint():
+ """Each template is fingerprinted separately, or editing one would falsely
+ mark the other's pages stale and re-render the whole wiki."""
+ assert structural_fingerprint(FILE_PAGE_TEMPLATE) != structural_fingerprint(
+ SYMBOL_SPOTLIGHT_TEMPLATE
+ )
+ # A store written by the current release is stale under neither sweep.
+ parsed = [_parsed("src/a.py")]
+ assert (
+ stale_file_page_paths(_stored({"src/a.py": _current_hash("hash-of-bytes")}), parsed) == []
+ )
+ assert stale_spotlight_paths({"src/a.py": [_spotlight_hash("hash-of-bytes")]}, parsed) == []
+
+
+def test_a_style_switch_makes_every_spotlight_stale():
+ parsed = [_parsed("src/a.py")]
+ stored = {"src/a.py": [_spotlight_hash("hash-of-bytes")]}
+ assert stale_spotlight_paths(stored, parsed, style_fingerprint="caveman") == ["src/a.py"]
+
+
def test_a_custom_style_template_override_is_what_gets_fingerprinted(tmp_path):
"""A style shipping its own file_page.j2 must fingerprint that, not the base.
diff --git a/tests/unit/generation/test_styles.py b/tests/unit/generation/test_styles.py
index 30a847201..7c6bc8076 100644
--- a/tests/unit/generation/test_styles.py
+++ b/tests/unit/generation/test_styles.py
@@ -252,6 +252,56 @@ async def test_style_change_moves_a_file_page_reuse_hash(
assert base_page.metadata["render_key"] != other_page.metadata["render_key"]
+async def test_a_symbol_spotlight_stores_a_render_key(
+ sample_parsed_file, sample_graph, graph_metrics, sample_source_bytes
+):
+ """A spotlight has no model path either, so it needs the same salt.
+
+ Without a stored key there is nothing for a later run to disagree with, so
+ an improved spotlight template can never be shown not to have landed, and
+ the page keeps whatever it said when it was first written.
+ """
+ _, gen = _make_gen(GenerationConfig(), sample_source_bytes)
+ page = await gen.generate_symbol_spotlight(
+ sample_parsed_file.symbols[0],
+ sample_parsed_file,
+ graph_metrics["pagerank"],
+ sample_graph,
+ {sample_parsed_file.file_info.path: sample_source_bytes},
+ )
+
+ assert page.metadata.get("render_key")
+
+
+async def test_a_spotlight_render_key_moves_when_its_template_does(
+ sample_parsed_file, sample_graph, graph_metrics, sample_source_bytes
+):
+ """The key is only worth storing if a template edit actually moves it."""
+ from repowise.core.generation.page_generator.structural import (
+ SYMBOL_SPOTLIGHT_TEMPLATE,
+ structural_content_hash,
+ structural_fingerprint,
+ )
+
+ _, gen = _make_gen(GenerationConfig(), sample_source_bytes)
+ page = await gen.generate_symbol_spotlight(
+ sample_parsed_file.symbols[0],
+ sample_parsed_file,
+ graph_metrics["pagerank"],
+ sample_graph,
+ {sample_parsed_file.file_info.path: sample_source_bytes},
+ )
+
+ subject = sample_parsed_file.content_hash
+ current = structural_content_hash(subject, structural_fingerprint(SYMBOL_SPOTLIGHT_TEMPLATE))
+ older = structural_content_hash(
+ subject, structural_fingerprint(SYMBOL_SPOTLIGHT_TEMPLATE, source="# an older template")
+ )
+
+ assert page.metadata["render_key"] == current
+ assert page.metadata["render_key"] != older
+
+
async def test_onboarding_page_type_constant_matches_system_prompts():
"""ONBOARDING_PAGE_TYPE must match the key used by the onboarding generators."""
from repowise.core.generation.page_generator import SYSTEM_PROMPTS