Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): code excerpts in module page prompts (#2713)

Swati Ahuja54c9618dac6b
diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.py
index 8bcc115e6..12d51de95 100644
--- a/packages/core/src/repowise/core/generation/context/assembler.py
+++ b/packages/core/src/repowise/core/generation/context/assembler.py
@@ -44,6 +44,7 @@ from .graph_intelligence import (
extract_community_meta,
extract_heritage,
)
+from .module_excerpts import module_excerpts
from .token_budget import (
estimate_kg_tokens,
estimate_tokens,
@@ -58,6 +59,8 @@ log = structlog.get_logger(__name__)
_MAX_IMPORTS = 30
# Tokens of code excerpts a module page prompt carries; the Public API claims them first.
_MODULE_EXCERPT_BUDGET = 10_000
+# Public API names are always listed, even past the budget, up to this cap; the rest are counted.
+_MODULE_EXCERPT_HARD_CAP = 15_000
# Maximum top-files to include in repo overview
_MAX_TOP_FILES = 20
@@ -506,11 +509,14 @@ class ContextAssembler:
child_pages: list[dict] | None = None,
packages: list[dict] | None = None,
public_api: list[dict] | None = None,
+ parsed_files: dict[str, ParsedFile] | None = None,
+ source_map: dict[str, bytes] | None = None,
) -> ModulePageContext:
"""Assemble context for the module_page template.
*public_api* (see :func:`~.public_api.compute_public_api`) has first
claim on the excerpt budget; entries past it keep only their name.
+ With *parsed_files* and *source_map*, code excerpts take what it leaves.
"""
total_symbols = sum(len(fc.symbols) for fc in file_contexts)
public_symbols = sum(
@@ -601,7 +607,8 @@ class ContextAssembler:
# ``extract_heritage``; the per-file cap stays so one dense file cannot
# take every slot.
key_classes: list[dict] = []
- for fc in sorted(file_contexts, key=lambda fc: (-fc.pagerank_score, fc.file_path)):
+ by_rank = sorted(file_contexts, key=lambda fc: (-fc.pagerank_score, fc.file_path))
+ for fc in by_rank:
for h in fc.heritage[:_MAX_CLASSES_PER_FILE]:
key_classes.append(h)
key_classes = key_classes[:_MAX_KEY_CLASSES]
@@ -629,6 +636,15 @@ class ContextAssembler:
most_fixed_file,
) = self._module_git_enrichment(files, set(files), git_meta_map)
+ api_rows, api_cost = self._excerpt_public_api(public_api or [])
+ excerpts, declared_files = module_excerpts(
+ public_api or [],
+ parsed_files or {},
+ [fc.file_path for fc in by_rank],
+ source_map or {},
+ max(0, _MODULE_EXCERPT_BUDGET - api_cost),
+ )
+
return ModulePageContext(
title=title,
directories=directories,
@@ -653,7 +669,10 @@ class ContextAssembler:
is_rollup=is_rollup,
child_pages=child_pages or [],
packages=packages or [],
- public_api=self._excerpt_public_api(public_api or []),
+ public_api=api_rows,
+ public_api_omitted=len(public_api or ()) - len(api_rows),
+ code_excerpts=excerpts.rendered,
+ declared_files=declared_files,
hotspot_count=hotspot_count,
stable_count=stable_count,
single_owner_files=single_owner_files,
@@ -662,11 +681,27 @@ class ContextAssembler:
most_fixed_file=most_fixed_file,
)
- def _excerpt_public_api(self, api: list[dict]) -> list[dict]:
- """Every entry, excerpted (signature + doc line) in order while the budget lasts."""
- cost = lambda e: self._estimate_tokens(f"{e['signature']} {e['doc']}") # noqa: E731
- kept, _ = items_within_budget(api, 0, _MODULE_EXCERPT_BUDGET, cost)
- return kept + [{**e, "signature": "", "doc": ""} for e in api[len(kept) :]]
+ def _excerpt_public_api(self, api: list[dict]) -> tuple[list[dict], int]:
+ """Every name (to the hard cap), then signature + doc line while half the rest lasts.
+
+ Costs follow the rendered shape (bare names grouped under their file);
+ +1 per row so rounding cannot add up past a limit.
+ """
+ files: set[str] = set()
+
+ def name_cost(e: dict) -> int:
+ head = 0 if e["file"] in files else self._estimate_tokens(f"- `{e['file']}`: ")
+ files.add(e["file"])
+ return 1 + head + self._estimate_tokens(f"`{e['name']}`, ")
+
+ def row_cost(e: dict) -> int:
+ row = f"- `{e['name']}` ({e['kind']}, `{e['file']}`): `` {e['signature']} `` {e['doc']}"
+ return 1 + self._estimate_tokens(row)
+
+ named, used = items_within_budget(api, 0, _MODULE_EXCERPT_HARD_CAP, name_cost)
+ half = used + max(0, _MODULE_EXCERPT_BUDGET - used) // 2
+ kept, used = items_within_budget(named, used, half, row_cost)
+ return kept + [{**e, "signature": "", "doc": ""} for e in named[len(kept) :]], used
def _package_boundaries(self, known_paths: set[str]) -> set[str]:
"""Package roots for this repo, resolved once.
diff --git a/packages/core/src/repowise/core/generation/context/contexts.py b/packages/core/src/repowise/core/generation/context/contexts.py
index 00af364a1..c4633a416 100644
--- a/packages/core/src/repowise/core/generation/context/contexts.py
+++ b/packages/core/src/repowise/core/generation/context/contexts.py
@@ -125,6 +125,11 @@ class ModulePageContext:
# What the page's packages publish, computed from manifests and re-exports:
# [{"name", "kind", "file", "signature", "doc", "alias_of"}]; past the budget, names only.
public_api: list[dict] = field(default_factory=list)
+ # Public API entries past the hard cap, counted, not listed.
+ public_api_omitted: int = 0
+ # Rendered source excerpts, then one-line signatures for files with no excerpt.
+ code_excerpts: str = ""
+ declared_files: list[str] = field(default_factory=list)
# Git-derived subsystem health, aggregated over the page's member files.
# All degrade to zero/empty when no git metadata is available, so the
# template renders nothing rather than a wrong number.
diff --git a/packages/core/src/repowise/core/generation/context/evidence.py b/packages/core/src/repowise/core/generation/context/evidence.py
index 385e5e7ac..0248d9c6a 100644
--- a/packages/core/src/repowise/core/generation/context/evidence.py
+++ b/packages/core/src/repowise/core/generation/context/evidence.py
@@ -220,6 +220,7 @@ def _resolve_reference_excerpt(
reference: str,
source_map: Mapping[str, bytes],
parsed_by_path: Mapping[str, object],
+ max_lines: int | None = None,
) -> tuple[_ExactExcerpt | None, str | None]:
"""Resolve a ``path::symbol`` reference to a framed excerpt, or return a skip reason."""
if "::" not in reference:
@@ -248,9 +249,14 @@ def _resolve_reference_excerpt(
end_line = min(end_line, len(lines))
if end_line < start_line:
return None, "invalid_line_range"
+ capped = max_lines is not None and end_line - start_line + 1 > max_lines
+ if capped:
+ end_line = start_line + max_lines - 1
body = "".join(lines[start_line - 1 : end_line])
if not body.strip():
return None, "empty_excerpt"
+ if capped:
+ body = body.rstrip("\n") + "\n" + _TRUNCATED
body = _EVIDENCE_FRAME_TAG.sub(lambda match: escape(match.group(), quote=False), body)
return (path, reference, start_line, end_line, body), None
@@ -259,6 +265,7 @@ def _eligible_reference_excerpts(
references: Sequence[str],
source_map: Mapping[str, bytes],
parsed_by_path: Mapping[str, object],
+ max_lines: int | None = None,
) -> tuple[list[_ExactExcerpt], list[EvidenceSkip]]:
"""Resolve every reference in order, partitioning into eligible excerpts and skips."""
eligible: list[_ExactExcerpt] = []
@@ -270,7 +277,9 @@ def _eligible_reference_excerpts(
skipped.append(EvidenceSkip(reference, "duplicate_reference"))
continue
seen.add(reference)
- excerpt, reason = _resolve_reference_excerpt(reference, source_map, parsed_by_path)
+ excerpt, reason = _resolve_reference_excerpt(
+ reference, source_map, parsed_by_path, max_lines
+ )
if reason is not None:
skipped.append(EvidenceSkip(reference, reason))
else:
@@ -327,20 +336,52 @@ def _render_exact_excerpts(
return included, blocks, skipped
-def _select_symbol_reference_evidence(
+def _keep_whole_excerpts(
+ eligible: list[_ExactExcerpt],
+ hard_char_limit: int,
+) -> tuple[list[EvidenceItem], list[str], list[EvidenceSkip]]:
+ """Keep line-capped excerpts whole, in priority order, up to the first that does not fit.
+
+ Stopping there keeps the result a priority prefix: a smaller, lower-priority
+ body never takes the place of a higher-priority one.
+ """
+ included: list[EvidenceItem] = []
+ blocks: list[str] = []
+ skipped: list[EvidenceSkip] = []
+ used = len(_EXACT_HEADER)
+ for index, (path, reference, start_line, end_line, body) in enumerate(eligible):
+ truncated = body.endswith(_TRUNCATED)
+ block = _source_wrapper(path, reference, start_line, end_line, body, truncated=truncated)
+ if used + len(block) > hard_char_limit:
+ skipped.extend(EvidenceSkip(ref, "budget_too_small") for _, ref, *_ in eligible[index:])
+ break
+ used += len(block)
+ included.append(EvidenceItem(path, body, truncated, reference, start_line, end_line))
+ blocks.append(block)
+ return included, blocks, skipped
+
+
+def select_symbol_evidence(
source_map: Mapping[str, bytes],
references: Sequence[str],
parsed_files: Sequence[object],
*,
token_budget: int,
+ max_lines: int | None = None,
) -> EvidenceSelection:
- """Select bounded exact bodies for symbol references, with skip provenance."""
+ """Select bounded exact bodies for symbol references, with skip provenance.
+
+ With *max_lines*, each body is capped to that many lines and kept whole in
+ reference order while it fits, instead of every body sharing the budget.
+ """
parsed_by_path = {
parsed.file_info.path: parsed
for parsed in parsed_files
if getattr(getattr(parsed, "file_info", None), "path", None)
}
- eligible, skipped = _eligible_reference_excerpts(references, source_map, parsed_by_path)
+ eligible, skipped = _eligible_reference_excerpts(
+ references, source_map, parsed_by_path, max_lines
+ )
if not eligible:
return EvidenceSelection(skipped=tuple(skipped))
@@ -349,6 +390,9 @@ def _select_symbol_reference_evidence(
return EvidenceSelection(skipped=tuple(skipped))
hard_char_limit = token_budget * 4 + 3
+ if max_lines is not None:
+ included, blocks, render_skips = _keep_whole_excerpts(eligible, hard_char_limit)
+ return _exact_selection(included, blocks, skipped + render_skips, token_budget)
selected, budget_skips = _drop_references_over_budget(eligible, hard_char_limit)
skipped.extend(budget_skips)
if not selected:
@@ -365,8 +409,15 @@ def _select_symbol_reference_evidence(
else _MIN_TRUNCATED_CONTENT * len(selected)
)
included, blocks, render_skips = _render_exact_excerpts(selected, remaining_chars)
- skipped.extend(render_skips)
+ return _exact_selection(included, blocks, skipped + render_skips, token_budget)
+
+def _exact_selection(
+ included: list[EvidenceItem],
+ blocks: list[str],
+ skipped: list[EvidenceSkip],
+ token_budget: int,
+) -> EvidenceSelection:
if not included:
return EvidenceSelection(skipped=tuple(skipped))
rendered = _EXACT_HEADER + "".join(blocks)
@@ -396,7 +447,7 @@ def _select_reference_evidence(
if not symbol_references:
return select_source_evidence(source_map, file_references, token_budget=token_budget)
if not file_references:
- return _select_symbol_reference_evidence(
+ return select_symbol_evidence(
source_map,
symbol_references,
parsed_files,
@@ -409,7 +460,7 @@ def _select_reference_evidence(
file_references,
token_budget=max(0, token_budget - symbol_budget - 1),
)
- symbols = _select_symbol_reference_evidence(
+ symbols = select_symbol_evidence(
source_map,
symbol_references,
parsed_files,
diff --git a/packages/core/src/repowise/core/generation/context/module_excerpts.py b/packages/core/src/repowise/core/generation/context/module_excerpts.py
new file mode 100644
index 000000000..cfa81725d
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/context/module_excerpts.py
@@ -0,0 +1,93 @@
+"""Code excerpts for a module page prompt, so the model reads code, not only paths.
+
+Candidates are the Public API first, then each file's public or documented
+top-level symbols by file PageRank, at most three per file. Each file with
+candidates costs one line of signatures, reserved first (up to half the
+budget); bodies (signature, docstring and code, up to 60 lines) get the rest,
+so bodies are what a tight budget drops first. A file whose symbols got no
+body keeps its line while the budget lasts.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from .evidence import EvidenceSelection, select_symbol_evidence
+from .token_budget import estimate_tokens, items_within_budget
+
+_PER_FILE = 3
+_MAX_LINES = 60
+
+
+def _file_of(symbol: Any) -> str:
+ return symbol.id.split("::", 1)[0]
+
+
+def _symbol(parsed: Mapping[str, Any], path: str, symbol_id: str) -> Any | None:
+ pf = parsed.get(path)
+ # An overload's implementation shares its declarations' id; serve the implementation.
+ hits = [s for s in (pf.symbols if pf else ()) if s.id == symbol_id]
+ return min(hits, key=lambda s: s.is_declaration, default=None)
+
+
+def excerpt_candidates(
+ public_api: Sequence[dict], parsed: Mapping[str, Any], ranked_paths: Sequence[str]
+) -> list[Any]:
+ """Symbols to excerpt, best first: the Public API, then ranked files' symbols."""
+ out: list[Any] = []
+ seen: set[str] = set()
+ for e in public_api:
+ sym = _symbol(parsed, e["file"], e["id"]) if e.get("id") else None
+ if sym is not None and sym.id not in seen:
+ out.append(sym)
+ seen.add(sym.id)
+ for path in ranked_paths:
+ room = _PER_FILE - sum(1 for s in out if _file_of(s) == path)
+ fresh = [s for s in _worth_excerpting(parsed.get(path)) if s.id not in seen]
+ for s in fresh[: max(0, room)]:
+ out.append(s)
+ seen.add(s.id)
+ return out
+
+
+def _worth_excerpting(pf: Any) -> list[Any]:
+ """A file's top-level symbols that are public or documented, in file order."""
+ symbols = pf.symbols if pf is not None else ()
+ return [
+ s for s in symbols if s.parent_name is None and (s.visibility == "public" or s.docstring)
+ ]
+
+
+def _declared_line(path: str, symbols: list[Any]) -> str:
+ sigs = "; ".join(" ".join((s.signature or s.name).split()) for s in symbols[:_PER_FILE])
+ return f"`{path}`: `` {sigs} ``"
+
+
+def module_excerpts(
+ public_api: Sequence[dict],
+ parsed: Mapping[str, Any],
+ ranked_paths: Sequence[str],
+ source_map: Mapping[str, bytes],
+ budget: int,
+) -> tuple[EvidenceSelection, list[str]]:
+ """Code excerpts and one-line declarations for the files left without one."""
+ candidates = excerpt_candidates(public_api, parsed, ranked_paths)
+ by_file: dict[str, list[Any]] = {}
+ for s in candidates:
+ by_file.setdefault(_file_of(s), []).append(s)
+ lines = {path: _declared_line(path, syms) for path, syms in by_file.items()}
+ # Signatures outrank bodies, but claim at most half so central files still show code.
+ _, reserved = items_within_budget(lines.values(), 0, budget // 2, estimate_tokens)
+ body_budget = budget - reserved
+ selection = select_symbol_evidence(
+ source_map,
+ [s.id for s in candidates],
+ [parsed[p] for p in by_file if p in parsed],
+ token_budget=body_budget,
+ max_lines=_MAX_LINES,
+ )
+ excerpted = {item.path for item in selection.included}
+ leftover = [line for path, line in lines.items() if path not in excerpted]
+ kept, _ = items_within_budget(leftover, selection.estimated_tokens, budget, estimate_tokens)
+ return selection, kept
diff --git a/packages/core/src/repowise/core/generation/context/public_api.py b/packages/core/src/repowise/core/generation/context/public_api.py
index 6c310010b..77610e79f 100644
--- a/packages/core/src/repowise/core/generation/context/public_api.py
+++ b/packages/core/src/repowise/core/generation/context/public_api.py
@@ -74,6 +74,7 @@ def _entry(name: str, sym: Any, path: str, alias_of: str) -> dict:
if sym is None: # a namespace export
return {
"name": name,
+ "id": "",
"kind": "module",
"file": path,
"signature": "",
@@ -83,6 +84,7 @@ def _entry(name: str, sym: Any, path: str, alias_of: str) -> dict:
doc = (sym.docstring or "").strip().splitlines()
return {
"name": name,
+ "id": sym.id,
"kind": sym.kind,
"file": path,
"signature": "" if alias_of else " ".join((sym.signature or "").split()),
@@ -165,7 +167,7 @@ class _Resolver:
def compute_public_api(
material: Iterable[str], parsed: Mapping[str, Any], package_roots: set[str]
) -> list[dict]:
- """``[{name, kind, file, signature, doc}]`` a page's files publish, entry order.
+ """``[{name, id, kind, file, signature, doc}]`` a page's files publish, entry order.
A symbol published under a second name (``export { A as B }``) is listed
again with ``alias_of`` and no excerpt. Ceiling: ``export { default as X }``,
diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.py
index ce4d9488f..f9ce3b83f 100644
--- a/packages/core/src/repowise/core/generation/page_generator/levels.py
+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py
@@ -331,6 +331,8 @@ def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
for path, count in package_file_counts(mg.file_paths, mg.packages)
],
public_api=compute_public_api(material, parsed_by_path, package_roots),
+ parsed_files=parsed_by_path,
+ source_map=run.source_map,
),
)
)
diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.py
index 27b892a4a..24ed6c956 100644
--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py
+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py
@@ -211,6 +211,8 @@ class PerTypeGenerationMixin:
owns_files: bool = True,
packages: list[dict] | None = None,
public_api: list[dict] | None = None,
+ parsed_files: dict[str, Any] | None = None,
+ source_map: dict[str, bytes] | None = None,
) -> GeneratedPage:
ctx = self._assembler.assemble_module_page(
title,
@@ -229,6 +231,8 @@ class PerTypeGenerationMixin:
child_pages=child_pages,
packages=packages,
public_api=public_api,
+ parsed_files=parsed_files,
+ source_map=source_map,
)
module_git_summary = None
if git_meta_map:
diff --git a/packages/core/src/repowise/core/generation/templates/module_page.j2 b/packages/core/src/repowise/core/generation/templates/module_page.j2
index ba1198e65..3ca0144f1 100644
--- a/packages/core/src/repowise/core/generation/templates/module_page.j2
+++ b/packages/core/src/repowise/core/generation/templates/module_page.j2
@@ -34,9 +34,15 @@ Name each in backticks by its path so it resolves to a wiki link; explain in one
{% if ctx.public_api %}
### Public API (computed from the package manifests and re-exports; what a consumer imports)
Name the central ones in backticks exactly as written and say what each is for; do not invent members beyond these.
-{% for e in ctx.public_api %}
-- `{{ e.name }}` ({{ e.kind }}, `{{ e.file }}`){% if e.alias_of %}, another name for `{{ e.alias_of }}`{% endif %}{% if e.signature %}: `` {{ e.signature }} ``{% endif %}{% if e.doc %}: {{ e.doc }}{% endif %}
+{% for e in ctx.public_api if e.signature %}
+- `{{ e.name }}` ({{ e.kind }}, `{{ e.file }}`): `` {{ e.signature }} ``{% if e.doc %}: {{ e.doc }}{% endif %}
{% endfor %}
+{% for file, rows in ctx.public_api | rejectattr("signature") | groupby("file") %}
+- `{{ file }}`: {% for e in rows %}`{{ e.name }}`{% if e.alias_of %} (another name for `{{ e.alias_of }}`){% endif %}{% if not loop.last %}, {% endif %}{% endfor %}
+{% endfor %}
+{% if ctx.public_api_omitted %}
+- and {{ ctx.public_api_omitted }} more public names not listed
+{% endif %}
{% endif %}
{% if ctx.entry_points %}
@@ -125,6 +131,14 @@ These modules change together with this one in history and nothing in the depend
Most active file: `{{ module_git_summary.most_active_file }}` ({{ module_git_summary.most_active_commits_90d }} commits in 90 days).
{% endif %}
+{% if ctx.declared_files %}
+### What other files declare (signatures only, no code shown)
+{% for line in ctx.declared_files %}
+- {{ line }}
+{% endfor %}
+{% endif %}
+{{ ctx.code_excerpts }}
+
## How to write this page
Write the page now from the material above. Requirements:
@@ -137,6 +151,8 @@ Write the page now from the material above. Requirements:
- Put any enumerable list (key files, dependencies, decisions) in a markdown table where it reads better than prose.
- Name the subsystem's key files and the neighbouring modules it consumes or serves in backticks exactly as written above so they resolve to wiki links. Mention a neighbour only where it explains this subsystem's role — no links for density, no reciprocal mentions for symmetry.
- Weave the true health and history signals (hotspots, bus factor, change coupling, fix history) into the narrative where they explain the subsystem's character. Skip any that are absent; never invent one.
+- State behaviour only when an excerpt shows it; otherwise say what the file declares.
+- A commit subject is history, not current behaviour: if you cite one, label it (historical, <date>).
- Third person throughout. No concluding or summary section.
- End with the mandatory `## Questions this page answers` section: 3 to 6 real questions this page answers, phrased as a developer or agent would ask them.
diff --git a/tests/unit/generation/test_module_code_excerpts.py b/tests/unit/generation/test_module_code_excerpts.py
new file mode 100644
index 000000000..9941bbedf
--- /dev/null
+++ b/tests/unit/generation/test_module_code_excerpts.py
@@ -0,0 +1,200 @@
+"""Code excerpts in module page prompts.
+
+The model reads code, not only paths: the Public API first, then the most
+central files' public or documented symbols, three per file. Bodies are capped
+at 60 lines and are what a tight budget drops first; a file left without one
+keeps a line of signatures. Public API names are always listed.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import networkx as nx
+import pytest
+
+from repowise.core.generation.context import assembler as assembler_mod
+from repowise.core.generation.context.evidence import select_symbol_evidence
+from repowise.core.generation.context.module_excerpts import module_excerpts
+from repowise.core.generation.context.token_budget import estimate_tokens
+from repowise.core.generation.context_assembler import ContextAssembler
+from repowise.core.generation.page_generator import PageGenerator
+from tests.unit.generation.test_package_public_api import (
+ _api,
+ _context,
+ _index,
+ _RecordingProvider,
+)
+
+_REPO = {
+ "packages/kit/pyproject.toml": '[project]\nname = "kit"\n',
+ "packages/kit/kit/__init__.py": "from .client import Client\n",
+ "packages/kit/kit/client.py": (
+ "class Client:\n"
+ ' """Sends requests."""\n\n'
+ " def send(self, body: str) -> int:\n"
+ " return len(body)\n"
+ ),
+ "packages/kit/kit/util.py": (
+ "def slugify(text: str) -> str:\n"
+ ' """Lower-case words joined by dashes."""\n'
+ ' return "-".join(text.lower().split())\n\n\n'
+ "def _hidden():\n pass\n"
+ ),
+}
+
+
+def _sources(repo: Path, parsed: dict) -> dict[str, bytes]:
+ return {p: (repo / p).read_bytes() for p in parsed}
+
+
+def _excerpt_share(ctx, config) -> int:
+ """Tokens of the rendered prompt's Public API rows, declarations and excerpts."""
+ prompt = PageGenerator(_RecordingProvider(), ContextAssembler(config), config)._render(
+ "module_page.j2", ctx=ctx
+ )
+ api = prompt[prompt.index("### Public API") :]
+ api = api[: api.index("\n##", 1)]
+ starts = [prompt.find(h) for h in ("### What other files declare", "## Exact source")]
+ end = prompt.index("## How to write")
+ code = prompt[min((i for i in starts if i >= 0), default=end) : end]
+ return estimate_tokens(api) + estimate_tokens(code)
+
+
+async def test_prompt_snapshot_has_public_api_excerpts_and_declarations(tmp_path, sample_config):
+ parsed = _index(tmp_path, _REPO)
+ provider = _RecordingProvider()
+ gen = PageGenerator(provider, ContextAssembler(sample_config), sample_config)
+ contexts = [_context("packages/kit/kit/util.py"), _context("packages/kit/kit/client.py")]
+ contexts[0].pagerank_score = 0.5
+ await gen.generate_module_page(
+ "Kit",
+ "python",
+ contexts,
+ nx.DiGraph(),
+ target_path="packages/kit",
+ public_api=_api(parsed, "packages/kit/"),
+ parsed_files=parsed,
+ source_map=_sources(tmp_path, parsed),
+ )
+ prompt = provider._calls[-1]["user_prompt"]
+ assert "State behaviour only when an excerpt shows it" in prompt
+ assert "(historical, <date>)" in prompt
+ # Nothing is left over at this size, so every candidate has its body.
+ assert "### What other files declare" not in prompt
+ # Fixtures are written in text mode, so a Windows checkout has CRLF sources.
+ code = prompt[prompt.index("## Exact source excerpts") : prompt.index("## How to write")]
+ code = code.replace("\r\n", "\n")
+ assert code.count("<source-excerpt ") == 2
+ # Public API first, then the higher-ranked file; private names never.
+ assert code.index('symbol="packages/kit/kit/client.py::Client"') < code.index(
+ 'symbol="packages/kit/kit/util.py::slugify"'
+ )
+ assert (
+ '<source-excerpt path="packages/kit/kit/util.py" '
+ 'symbol="packages/kit/kit/util.py::slugify" lines="1-3">\n'
+ "def slugify(text: str) -> str:\n"
+ ' """Lower-case words joined by dashes."""\n'
+ ' return "-".join(text.lower().split())\n\n'
+ "</source-excerpt>"
+ ) in code
+ assert "_hidden" not in code
+
+
+def _many_functions(n: int, body_lines: int) -> str:
+ body = "".join(f" x{i} = {i}\n" for i in range(body_lines))
+ return "".join(f"def f{k}():\n{body}\n\n" for k in range(n))
+
+
+async def test_budget_drops_bodies_first_and_keeps_every_public_name(
+ tmp_path, sample_config, monkeypatch
+):
+ files = {f"pkg/m{i}.py": _many_functions(3, 40) for i in range(20)}
+ files["pkg/__init__.py"] = "".join(f"from .m{i} import f0\n" for i in range(20))
+ files["pyproject.toml"] = '[project]\nname = "pkg"\n'
+ parsed = _index(tmp_path, files)
+ api = _api(parsed, "pkg/")
+ budget = 1_500
+ monkeypatch.setattr(assembler_mod, "_MODULE_EXCERPT_BUDGET", budget)
+ ctx = ContextAssembler(sample_config).assemble_module_page(
+ "Pkg",
+ "python",
+ [_context(p) for p in files if p.endswith(".py")],
+ nx.DiGraph(),
+ public_api=api,
+ parsed_files=parsed,
+ source_map=_sources(tmp_path, parsed),
+ )
+ assert [e["name"] for e in ctx.public_api] == [e["name"] for e in api]
+ # Section headings and instructions sit outside the budget; allow for them.
+ assert _excerpt_share(ctx, sample_config) <= budget + 200
+ assert 0 < ctx.code_excerpts.count("<source-excerpt ") < 20
+ # A file whose body was dropped keeps its signatures.
+ assert ctx.declared_files and all("def f0()" in line for line in ctx.declared_files)
+
+
+async def test_a_long_body_is_capped_at_sixty_lines(tmp_path, sample_config):
+ parsed = _index(tmp_path, {"big.py": _many_functions(1, 100)})
+ ctx = ContextAssembler(sample_config).assemble_module_page(
+ "Big",
+ "python",
+ [_context("big.py")],
+ nx.DiGraph(),
+ parsed_files=parsed,
+ source_map=_sources(tmp_path, parsed),
+ )
+ assert 'symbol="big.py::f0" lines="1-60" truncated="true"' in ctx.code_excerpts
+ assert "x59 = 59" not in ctx.code_excerpts and "x58 = 58" in ctx.code_excerpts
+
+
+def test_public_api_names_past_the_hard_cap_are_counted(sample_config):
+ api = [
+ {
+ "name": f"exported_name_{i}",
+ "kind": "function",
+ "file": "a.ts",
+ "signature": "x" * 400,
+ "doc": "",
+ "alias_of": "",
+ }
+ for i in range(6_000)
+ ]
+ ctx = ContextAssembler(sample_config).assemble_module_page(
+ "A", "typescript", [_context("a.ts")], nx.DiGraph(), public_api=api
+ )
+ assert ctx.public_api_omitted == len(api) - len(ctx.public_api) > 0
+ assert _excerpt_share(ctx, sample_config) <= assembler_mod._MODULE_EXCERPT_HARD_CAP
+
+
[email protected]("budget", [60, 200, 10_000])
+def test_line_capped_excerpts_stay_whole_in_priority_order(tmp_path, budget):
+ parsed = _index(tmp_path, {"m.py": _many_functions(4, 5)})
+ refs = [f"m.py::f{k}" for k in range(4)]
+ sel = select_symbol_evidence(
+ _sources(tmp_path, parsed),
+ refs,
+ list(parsed.values()),
+ token_budget=budget,
+ max_lines=60,
+ )
+ assert sel.estimated_tokens <= budget
+ assert not any(item.truncated for item in sel.included)
+ kept = [item.symbol for item in sel.included]
+ assert kept == refs[: len(kept)]
+ if budget == 10_000:
+ assert kept == refs
+
+
+def test_a_big_public_api_body_is_not_displaced_by_a_smaller_ranked_one(tmp_path):
+ parsed = _index(
+ tmp_path, {"big.py": _many_functions(1, 50), "small.py": "def g():\n return 1\n"}
+ )
+ sources = _sources(tmp_path, parsed)
+ api = [{"name": "f0", "id": "big.py::f0", "file": "big.py"}]
+ # Room for the small body alone, not the big one.
+ budget = 260
+ selection, declared = module_excerpts(api, parsed, ["small.py"], sources, budget)
+ alone, _ = module_excerpts([], parsed, ["small.py"], sources, budget)
+ assert [item.symbol for item in alone.included] == ["small.py::g"]
+ assert selection.included == ()
+ assert any("big.py" in line for line in declared)
diff --git a/tests/unit/test_no_unranked_truncation.py b/tests/unit/test_no_unranked_truncation.py
index e3a4ee51c..6377badd3 100644
--- a/tests/unit/test_no_unranked_truncation.py
+++ b/tests/unit/test_no_unranked_truncation.py
@@ -75,7 +75,7 @@ _CUTS = [
Cut(
"assemble_module_page.key_classes",
"core/src/repowise/core/generation/context/assembler.py",
- "for fc in sorted(file_contexts, key=lambda fc: (-fc.pagerank_score, fc.file_path)):",
+ "by_rank = sorted(file_contexts, key=lambda fc: (-fc.pagerank_score, fc.file_path))",
"the file's PageRank, before _MAX_KEY_CLASSES",
forbids="for fc in file_contexts:\n for h in fc.heritage",
),