Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): a model names the concept pages, and the tree stops documenting documentation (#1025)

Raghav Chamadiyaf88039132c4e
diff --git a/packages/core/src/repowise/core/generation/concept_tree/planner.py b/packages/core/src/repowise/core/generation/concept_tree/planner.py
index 73e29e3ee..6d13a1ef0 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/planner.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/planner.py
@@ -33,6 +33,7 @@ from .naming import (
decode_response,
deterministic_scope,
deterministic_title,
+ disambiguate_titles,
ground_scopes,
parse_json_object,
)
@@ -71,6 +72,17 @@ class PlannerInputs:
test_files: set[str] = field(default_factory=set)
+def _reasoning_kwargs(reasoning: str | None) -> dict[str, str]:
+ """Pass the run's reasoning setting through, or nothing when unset.
+
+ A user who turned thinking off did so for the whole run, and a naming call
+ that quietly re-enables it bills them for what they declined. Omitted
+ rather than defaulted when unset so a provider that does not take the
+ keyword is not handed one.
+ """
+ return {"reasoning": reasoning} if reasoning else {}
+
+
@contextlib.contextmanager
def _billed_as(provider: Any, operation: str) -> Iterator[None]:
"""Label this step's spend on the Costs page; no-op without a tracker."""
@@ -154,24 +166,15 @@ def _disambiguate_titles(named: list[NamedGroup]) -> None:
seen[entry.title.lower()] = 1
-def plan_deterministic(
- inputs: PlannerInputs, *, params: GroupingParams | None = None
-) -> tuple[ConceptOutline, list[ConceptGroup]]:
- """The keyless outline: real structure, names derived from paths and layers.
+def name_deterministically(groups: list[ConceptGroup], inputs: PlannerInputs) -> ConceptOutline:
+ """Title and section already-computed *groups* from paths and layers only.
- Per D5 this is the same tree the LLM path produces — identical grouping,
- identical identities — with plainer titles. Adding a key upgrades the
- prose, never the shape, so a user who adds one later does not get their
- wiki re-partitioned underneath them.
-
- That guarantee only holds if both paths partition with the same bounds, so
- *params* is threaded rather than re-derived. It was not, once: the caller's
- bounds were dropped here and the two paths produced different page counts
- for the same repository.
+ Split out from :func:`plan_deterministic` so a caller that has already
+ partitioned the files can name that exact partition rather than asking for
+ a second one. Two callers each running the grouper is two places that have
+ to agree about page identity, which is the arrangement D2 exists to
+ prevent.
"""
- groups = group_files(
- inputs.production_files, layer_of_file=inputs.layer_of_file, params=params
- )
named = [
NamedGroup(
group=g,
@@ -187,7 +190,26 @@ def plan_deterministic(
outline = _build_outline(named)
_merge_thin_sections(outline)
outline.naming_mode = "deterministic"
- return outline, groups
+ return outline
+
+
+def plan_deterministic(
+ inputs: PlannerInputs, *, params: GroupingParams | None = None
+) -> tuple[ConceptOutline, list[ConceptGroup]]:
+ """The keyless outline: real structure, names derived from paths and layers.
+
+ Per D5 this is the same tree the LLM path produces — identical grouping,
+ identical identities — with plainer titles. Adding a key upgrades the
+ prose, never the shape, so a user who adds one later does not get their
+ wiki re-partitioned underneath them.
+
+ That guarantee only holds if both paths partition with the same bounds, so
+ *params* is threaded rather than re-derived. It was not, once: the caller's
+ bounds were dropped here and the two paths produced different page counts
+ for the same repository.
+ """
+ groups = group_files(inputs.production_files, layer_of_file=inputs.layer_of_file, params=params)
+ return name_deterministically(groups, inputs), groups
_REPAIR_INSTRUCTIONS = """\
@@ -243,6 +265,20 @@ def _usable_scope(scope: str, target_path: str) -> bool:
return "/" not in text.split(" ", 1)[0] or " " in text.strip()
+def _force_unique_titles(outline: ConceptOutline) -> int:
+ """Qualify any title still shared by two pages. Returns how many moved."""
+ pages = outline.pages
+ unique = disambiguate_titles([(p.title, p.target_path) for p in pages])
+ changed = 0
+ for page, title in zip(pages, unique, strict=True):
+ if title != page.title:
+ page.title = title
+ changed += 1
+ if changed:
+ logger.info("concept_outline_titles_disambiguated", changed=changed)
+ return changed
+
+
def _repair_targets(outline: ConceptOutline, report: OutlineReport) -> set[str]:
"""The page titles worth a second call. Everything else is left alone.
@@ -267,18 +303,46 @@ async def plan_outline(
deterministic: bool = False,
params: GroupingParams | None = None,
repair: bool = True,
+ reasoning: str | None = None,
) -> tuple[ConceptOutline, OutlineReport]:
- """Produce a validated outline for *inputs*.
+ """Group *inputs* and produce a validated outline over that grouping."""
+ groups = group_files(inputs.production_files, layer_of_file=inputs.layer_of_file, params=params)
+ return await name_groups(
+ groups,
+ inputs,
+ provider=provider,
+ deterministic=deterministic,
+ params=params,
+ repair=repair,
+ reasoning=reasoning,
+ )
- Falls back to :func:`plan_deterministic` whenever there is no provider, the
- run is in deterministic mode, or the model's response cannot be used. A
- failed naming call costs the wiki its titles, never its structure.
+
+async def name_groups(
+ groups: list[ConceptGroup],
+ inputs: PlannerInputs,
+ *,
+ provider: Any | None = None,
+ deterministic: bool = False,
+ params: GroupingParams | None = None,
+ repair: bool = True,
+ reasoning: str | None = None,
+) -> tuple[ConceptOutline, OutlineReport]:
+ """Name and section an already-computed partition, then validate it.
+
+ One call names every group. The model receives opaque ids and returns a
+ title, a scope and a section per id; it never decides membership, so a
+ response that is late, partial, malformed or full of invented ids costs
+ the wiki its titles and never its structure or its coverage.
+
+ Falls back to :func:`name_deterministically` whenever there is no provider,
+ the run is in deterministic mode, or the model's response cannot be used.
"""
all_files = set(inputs.production_files)
resolved = params or params_for(len(all_files))
if deterministic or provider is None:
- outline, _groups = plan_deterministic(inputs, params=params)
+ outline = name_deterministically(groups, inputs)
report = validate_outline(
outline,
all_files=all_files,
@@ -287,9 +351,6 @@ async def plan_outline(
)
return outline, report
- groups = group_files(
- inputs.production_files, layer_of_file=inputs.layer_of_file, params=params
- )
payload, index = build_payload(
groups,
layer_labels=inputs.layer_labels,
@@ -333,6 +394,7 @@ async def plan_outline(
user_prompt=instructions + body,
max_tokens=16000,
temperature=0.2,
+ **_reasoning_kwargs(reasoning),
)
data = parse_json_object(getattr(response, "content", "") or "")
except Exception as exc:
@@ -343,9 +405,7 @@ async def plan_outline(
# and could take the run down — the one failure mode this design exists to
# rule out. Decoding is defensive in its own right; this is the second line.
try:
- named, invented_ids = decode_response(
- data, index, layer_labels=inputs.layer_labels
- )
+ named, invented_ids = decode_response(data, index, layer_labels=inputs.layer_labels)
except Exception as exc:
logger.warning("concept_outline_decode_failed", error=str(exc))
named, invented_ids = decode_response({}, index, layer_labels=inputs.layer_labels)
@@ -369,7 +429,7 @@ async def plan_outline(
if repair:
targets = _repair_targets(outline, report)
if targets:
- await _repair_titles(outline, index, targets, provider=provider)
+ await _repair_titles(outline, index, targets, provider=provider, reasoning=reasoning)
# Repair writes new prose, so it has to face the same citation
# check the first pass did. Grounding after the last write rather
# than after the first is the difference between checking the page
@@ -383,6 +443,21 @@ async def plan_outline(
)
report.invented_paths = sorted(set(report.invented_paths) | set(ungrounded))
+ # Last line of defence on titles. Repair is a second model call and can
+ # decline, run out of ids, or hand back a name already in use, so a
+ # duplicate can still reach here. Identity is structural, so nothing is
+ # lost when two titles collide, but a reader sees two identical rows and
+ # cannot tell which is which. Qualifying them by path is the same rule the
+ # deterministic path already applies, and it cannot fail.
+ if _force_unique_titles(outline):
+ report = validate_outline(
+ outline,
+ all_files=all_files,
+ test_files=inputs.test_files,
+ max_files_per_page=resolved.max_files,
+ )
+ report.invented_paths = sorted(set(report.invented_paths) | set(ungrounded))
+
logger.info(
"concept_outline_planned",
pages=report.page_count,
@@ -390,6 +465,11 @@ async def plan_outline(
coverage=round(report.coverage, 4),
invented=len(report.invented_paths),
naming_mode=outline.naming_mode,
+ # How many titles the model actually decided. ``naming_mode`` only says
+ # that at least one did, so a run where the response was unusable and
+ # every page fell back to its path still reads as "llm". That happened
+ # and looked like success; this is the number that shows it.
+ named_by_model=sum(1 for p in outline.pages if p.named_by_model),
)
return outline, report
@@ -400,6 +480,7 @@ async def _repair_titles(
targets: set[str],
*,
provider: Any,
+ reasoning: str | None = None,
) -> None:
"""Re-ask for just the failing titles, then apply only what improved.
@@ -436,13 +517,22 @@ async def _repair_titles(
taken="\n".join(f"- {t}" for t in taken) or "(none)",
failures="\n".join(lines),
)
+ # Budgeted per failing group rather than flat. Repair was written for the
+ # handful a long request loses near the end of the list, and a flat 2000
+ # tokens covers that. It does not cover the case that actually matters: a
+ # first call that comes back with no usable names at all, where repair is
+ # asked to name the whole repository and truncates, so an outline that
+ # could have been recovered ships with every title derived from a path.
+ # Observed on this repository at 82 groups.
+ budget = max(2000, min(16000, 200 + 150 * len(failing)))
try:
with _billed_as(provider, COST_OPERATION):
response = await provider.generate(
system_prompt=SYSTEM_PROMPT,
user_prompt=prompt,
- max_tokens=2000,
+ max_tokens=budget,
temperature=0.2,
+ **_reasoning_kwargs(reasoning),
)
data = parse_json_object(getattr(response, "content", "") or "")
except Exception as exc:
diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.py
index c193e3662..aac397cce 100644
--- a/packages/core/src/repowise/core/generation/page_generator/levels.py
+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py
@@ -311,6 +311,8 @@ def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
target_path=mg.key,
structural_key=mg.structural_key,
members=list(mg.file_paths),
+ section=mg.section,
+ order=mg.order,
),
)
)
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index 78657045d..fbfb2387b 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -22,6 +22,7 @@ from ..context_assembler import FilePageContext
from ..models import (
STRUCTURALLY_KEYED_PAGE_TYPES,
GeneratedPage,
+ compute_page_id,
member_structural_key,
)
from . import levels as _levels
@@ -212,7 +213,7 @@ class _GenerationRun:
already_completed=len(self.completed_ids),
)
- def _compute_selection(self) -> None:
+ async def _compute_selection(self) -> None:
"""Run the selection subsystem and derive the level allow-sets."""
code_files = [
p
@@ -278,6 +279,14 @@ class _GenerationRun:
self.sel_module_groups = list(selection.module_groups)
self.sel_scc_groups = list(selection.scc_groups)
+ # One model call names every concept group. It runs here, after the
+ # partition exists and before any page id is computed, because naming
+ # is the one part of the concept tree that is judgement rather than
+ # structure: the grouper decides what a page covers, the model decides
+ # what to call it. Kept out of ``select_pages`` so the cost estimator
+ # and scope resolution, which call it too, stay free of the model.
+ await self._name_concept_groups()
+
# Tiered doc generation: split the selected file pages into a full-LLM
# tier-1 and a deterministic template-only tier-2. When tier1_top_n is
# None this puts every selected page in tier-1 (no behaviour change).
@@ -308,6 +317,121 @@ class _GenerationRun:
),
)
+ async def _name_concept_groups(self) -> None:
+ """Give every concept group a title, a scope and a section, in one call.
+
+ The model receives opaque group ids and returns a name per id. It never
+ sees the membership question, so the two failures the probes measured
+ have no entry point here: a group it skips keeps the deterministic
+ title selection already gave it, and an id it invents is discarded.
+
+ Both the keyless and the named path read the *same* partition, because
+ selection computed it once and this only relabels it. D5's guarantee
+ that adding a key changes the prose and never the shape is therefore
+ structural rather than a property two code paths have to agree on.
+
+ Every failure mode degrades to those deterministic titles rather than
+ failing the run: no provider, deterministic mode, a provider that
+ raises, a response that will not parse, or a group the model left out.
+ """
+ groups = list(getattr(self.selection, "concept_groups", None) or [])
+ deterministic = bool(getattr(self.config, "deterministic", False))
+ provider = getattr(self.gen, "_provider", None)
+ if not groups or deterministic or provider is None:
+ return
+
+ # Naming is only worth a call if this run is going to write a concept
+ # page with the result. Three runs reach here and do not: an
+ # incremental update sets ``file_pages_only`` and returns before level
+ # 4, a scoped run may have asked for pages of other types only, and a
+ # resumed run may already hold every concept page. Each would otherwise
+ # buy a whole-repository outline and discard it, which on a post-commit
+ # update hook is a bill per commit for nothing.
+ if getattr(self.config, "file_pages_only", False):
+ return
+ if not any(self._emit(compute_page_id("module_page", mg.key)) for mg in self.sel_module_groups):
+ return
+
+ from ..concept_tree.planner import PlannerInputs, name_groups
+
+ inputs = PlannerInputs(
+ repo_name=self.repo_name or "",
+ # The grouping is total, so the union of the members is exactly the
+ # file set the grouper partitioned. Rebuilt from the groups rather
+ # than re-filtered from ``parsed_files`` so the validator measures
+ # coverage against the set that was actually grouped.
+ production_files=[m for g in groups for m in g.members],
+ repo_root=Path(self.repo_path) if self.repo_path else None,
+ layer_labels=dict(getattr(self.selection, "layer_labels", None) or {}),
+ entry_points={
+ p.file_info.path
+ for p in self.parsed_files
+ if getattr(p.file_info, "is_entry_point", False)
+ },
+ )
+
+ try:
+ outline, report = await name_groups(
+ groups,
+ inputs,
+ provider=provider,
+ reasoning=getattr(self.config, "reasoning", None),
+ )
+ except Exception as exc:
+ # ``name_groups`` guards the call and the decode itself, so reaching
+ # here means a defect rather than a bad response. The titles from
+ # selection are already correct, so the wiki is worse-named and not
+ # broken, and that is the trade this whole step is allowed to make.
+ log.warning("concept_naming.failed", error=str(exc))
+ return
+
+ title_of: dict[str, str] = {}
+ section_of: dict[str, str] = {}
+ order_of: dict[str, int] = {}
+ position = 0
+ for section in outline.sections:
+ for page in section.pages:
+ title_of[page.structural_key] = page.title
+ section_of[page.structural_key] = section.title
+ order_of[page.structural_key] = position
+ position += 1
+
+ from dataclasses import replace as _replace
+
+ renamed = 0
+ groups_out = []
+ for mg in self.sel_module_groups:
+ title = title_of.get(mg.structural_key)
+ if title is None:
+ # The outline covers every group it was given, so this is a
+ # group that never reached the namer. It keeps what it had.
+ groups_out.append(mg)
+ continue
+ if title != mg.display:
+ renamed += 1
+ groups_out.append(
+ _replace(
+ mg,
+ display=title,
+ section=section_of.get(mg.structural_key, ""),
+ order=order_of.get(mg.structural_key, 0),
+ )
+ )
+ # Generation order is left alone: it is summed PageRank, so the most
+ # central subsystem is written first and lands in the store earliest.
+ # Where the reader meets each page is ``order``, applied by the tree.
+ self.sel_module_groups = groups_out
+
+ log.info(
+ "concept_naming.applied",
+ groups=len(groups),
+ renamed=renamed,
+ sections=len(outline.sections),
+ naming_mode=outline.naming_mode,
+ duplicate_titles=len(report.duplicate_titles),
+ invented=len(report.invented_paths),
+ )
+
def _compute_demand(self) -> dict[str, int]:
"""Per-file question demand for the FAQ-weighted budget tilt.
@@ -574,7 +698,7 @@ class _GenerationRun:
async def execute(self) -> list[GeneratedPage]:
self._setup_job()
await self._seed_resume()
- self._compute_selection()
+ await self._compute_selection()
self._announce_total()
self._compute_ia()
diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.py
index 6d2b1b1a1..ee3d4ac11 100644
--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py
+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py
@@ -110,6 +110,8 @@ class PerTypeGenerationMixin:
target_path: str | None = None,
structural_key: str = "",
members: list[str] | None = None,
+ section: str = "",
+ order: int = 0,
) -> GeneratedPage:
ctx = self._assembler.assemble_module_page(
title,
@@ -164,11 +166,30 @@ class PerTypeGenerationMixin:
# dropped files would be parented somewhere else entirely, and the two
# records of what the page covers would disagree.
covered = sorted(members) if members else sorted(fc.file_path for fc in file_contexts)
- if self._config.deterministic:
- page = self._deterministic_module_page(ctx, page_target, title, module_git_summary)
+
+ def _stamp_concept(page: GeneratedPage) -> GeneratedPage:
+ """Record where the namer put this page in the reading order.
+
+ Only stamped when a namer ran, so an absent key means "nobody
+ decided", which is what the tree needs to tell apart from "first".
+ Display only: the tree reads ``concept_order`` to order siblings,
+ and neither field takes part in page identity, so re-sectioning a
+ wiki renumbers it and mints no new pages.
+ """
page.metadata["file_paths"] = covered
+ if section:
+ page.metadata["concept_section"] = section
+ page.metadata["concept_order"] = order
+ # Set by the producer, preserved by ``_stamp_structural_keys``. The
+ # grouper hashed exactly this member list when it decided what the
+ # group was, so recomputing it downstream would give two places
+ # that must agree about page identity — the arrangement D2 rules out.
page.structural_key = structural_key or page.structural_key
return page
+
+ if self._config.deterministic:
+ page = self._deterministic_module_page(ctx, page_target, title, module_git_summary)
+ return _stamp_concept(page)
user_prompt = self._render("module_page.j2", ctx=ctx, module_git_summary=module_git_summary)
response = await self._call_provider(
"module_page", user_prompt, str(uuid.uuid4()), target_path=page_target
@@ -181,13 +202,7 @@ class PerTypeGenerationMixin:
compute_source_hash(user_prompt),
GENERATION_LEVELS["module_page"],
)
- page.metadata["file_paths"] = covered
- # Set by the producer, preserved by ``_stamp_structural_keys``. The
- # grouper hashed exactly this member list when it decided what the
- # group was, so recomputing it downstream would give two places that
- # must agree about page identity — the arrangement D2 rules out.
- page.structural_key = structural_key or page.structural_key
- return page
+ return _stamp_concept(page)
async def generate_scc_page(
self,
diff --git a/packages/core/src/repowise/core/generation/page_tree.py b/packages/core/src/repowise/core/generation/page_tree.py
index 45eac9956..a184ec5c5 100644
--- a/packages/core/src/repowise/core/generation/page_tree.py
+++ b/packages/core/src/repowise/core/generation/page_tree.py
@@ -102,6 +102,8 @@ def _sort_key(page: GeneratedPage, layer_rank: dict[str, int]) -> tuple:
rank = _onboarding_rank(page)
elif page.page_type == "layer_page":
rank = layer_rank.get(page.target_path, len(layer_rank))
+ elif page.page_type == "module_page":
+ rank = _concept_rank(page)
else:
rank = 0
return (
@@ -112,6 +114,29 @@ def _sort_key(page: GeneratedPage, layer_rank: dict[str, int]) -> tuple:
)
+def _concept_rank(page: GeneratedPage) -> int:
+ """Where the namer put this concept page in the reading order.
+
+ A concept page's siblings are the other concept pages under the same
+ layer, and sorting them by target path puts the reader wherever the
+ alphabet lands. The namer already decided an order as a narrative, so use
+ it when it exists.
+
+ Pages with no order come from a run that never named them (keyless, or a
+ wiki written before naming shipped) and sort after the named ones, where
+ the tie-break on ``target_path`` gives them the order they have today.
+ Deliberately not zero: that is a real position, and borrowing it would
+ interleave unnamed pages through the narrative.
+ """
+ order = page.metadata.get("concept_order")
+ return order if isinstance(order, int) else _UNRANKED_CONCEPT
+
+
+# Larger than any real position, so unnamed pages sort after named ones for
+# any outline the grouper's size ladder can produce.
+_UNRANKED_CONCEPT = 1_000_000
+
+
def _dominant_layer(paths: Iterable[str], layer_of_file: dict[str, str]) -> str:
"""The layer most of these files belong to.
diff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.py
index a854fd56f..cf8d0a60f 100644
--- a/packages/core/src/repowise/core/generation/selection/selector.py
+++ b/packages/core/src/repowise/core/generation/selection/selector.py
@@ -17,7 +17,7 @@ import structlog
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
-from ..concept_tree.grouping import group_files
+from ..concept_tree.grouping import ConceptGroup, group_files
from ..concept_tree.naming import deterministic_title, disambiguate_titles
from ..models import scc_page_slug
from ..tour import DEFAULT_MAX_LANDMARKS, tour_landmark_paths
@@ -42,6 +42,13 @@ _INFRA_LANGUAGES = _LANG_REGISTRY.infra_languages()
_INFRA_FILENAMES = frozenset({"Dockerfile", "Makefile", "GNUmakefile"})
_CODE_LANGUAGES = _LANG_REGISTRY.code_languages()
+# Top-level directories whose contents document or illustrate the repository
+# rather than being it. Matched as a whole first path segment only. See
+# ``_is_support_file`` for why the anchoring is the point.
+_SUPPORT_ROOT_DIRS = frozenset(
+ {"docs", "doc", "documentation", "docs_src", "examples", "example", "samples", "sample"}
+)
+
# ---------------------------------------------------------------------------
# Output dataclasses
@@ -72,6 +79,26 @@ class ModuleGroup:
label: str | None = None
cohesion: float | None = None
structural_key: str = ""
+ #: The section this page belongs to, and its position in the reading order
+ #: the namer chose. Display only: neither takes part in page identity, so a
+ #: re-run that re-sections the wiki renumbers it and mints nothing.
+ section: str = ""
+ order: int = 0
+
+
+@dataclass
+class ConceptCandidates:
+ """The concept partition, in both the shapes the run needs.
+
+ ``scored`` is what the budget and the generator consume. ``groups`` is the
+ partition itself, carried out so the namer can title the exact groups that
+ were computed here instead of asking the grouper for a second partition
+ and hoping the two agree.
+ """
+
+ scored: list[tuple[float, ModuleGroup]] = field(default_factory=list)
+ groups: list[ConceptGroup] = field(default_factory=list)
+ layer_labels: dict[str, str] = field(default_factory=dict)
@dataclass
@@ -93,6 +120,12 @@ class Selection:
# (Phase G coverage tail). Kept separate from ``file_page_paths`` so cost
# estimation stays honest (these are free) and the CLI can report the split.
deterministic_tail_paths: list[str] = field(default_factory=list)
+ # The concept partition behind ``module_groups``, one entry per group, plus
+ # the layer names the grouper steered by. Carried so a caller holding a
+ # provider can name these groups; selection itself stays free of the model
+ # so the cost estimator and scope resolution keep costing nothing.
+ concept_groups: list[ConceptGroup] = field(default_factory=list)
+ layer_labels: dict[str, str] = field(default_factory=dict)
def counts(self) -> dict[str, int]:
"""Per-page-type counts of BUDGETED (LLM-costed) pages.
@@ -305,7 +338,28 @@ def _layer_map_from_kg(
return layer_of_file, labels
-def _build_module_groups(inputs: SelectionInputs) -> list[tuple[float, ModuleGroup]]:
+def _is_support_file(path: str) -> bool:
+ """Whether *path* is documentation or example source rather than the subject.
+
+ Anchored at the repository root and matched on whole path segments, which
+ is the difference between this rule and the substring match the test rule
+ originally shipped with. A package that has a ``docs`` directory as a
+ *feature* keeps its pages; a ``docs/`` tree at the top of the repository
+ does not, because it is the subject's documentation and not the subject.
+
+ Measured rather than assumed, the same way the test exclusion was: on one
+ web framework 10 of 15 concept pages documented the example snippets that
+ illustrate the library while the library itself got 4. These files match a
+ concept query on vocabulary without answering how the system works, and a
+ wiki that documents its subject's documentation has said nothing about the
+ subject. They keep their deterministic file pages, so nothing becomes
+ unreachable.
+ """
+ head = path.split("/", 1)[0].lower()
+ return head in _SUPPORT_ROOT_DIRS
+
+
+def _build_module_groups(inputs: SelectionInputs) -> ConceptCandidates:
"""Return the concept groups, scored, one per ``module_page``.
This is the concept tree, not one page per directory. The partition comes
@@ -326,13 +380,27 @@ def _build_module_groups(inputs: SelectionInputs) -> list[tuple[float, ModuleGro
score no longer rations anything — see :func:`select_pages` — but page
order is still what the reader sees first.
"""
- files = [
+ production = [
p.file_info.path
for p in inputs.parsed_files
if _is_code_file(p) and not getattr(p.file_info, "is_test", False)
]
+ files = [p for p in production if not _is_support_file(p)]
if not files:
- return []
+ # A repository whose production code is entirely under one of those
+ # roots. Documenting the docs is a bad wiki; having no wiki at all is
+ # a worse one, so the floor yields rather than emptying the tree.
+ if production:
+ log.info("page_selection.support_floor_skipped", files=len(production))
+ files = production
+ elif len(files) != len(production):
+ log.info(
+ "page_selection.support_floor",
+ excluded=len(production) - len(files),
+ kept=len(files),
+ )
+ if not files:
+ return ConceptCandidates()
layer_of_file, layer_labels = _layer_map_from_kg(inputs)
groups = group_files(files, layer_of_file=layer_of_file)
@@ -372,7 +440,7 @@ def _build_module_groups(inputs: SelectionInputs) -> list[tuple[float, ModuleGro
)
)
scored.sort(key=lambda x: (-x[0], x[1].key))
- return scored
+ return ConceptCandidates(scored=scored, groups=groups, layer_labels=layer_labels)
def _build_api_candidates(inputs: SelectionInputs) -> list[tuple[float, str]]:
@@ -537,7 +605,7 @@ def _ensure_landmarks(selected: list[str], landmarks: list[str]) -> list[str]:
def _select_everything(
files: list,
symbols: list,
- modules: list,
+ concepts: ConceptCandidates,
apis: list,
infras: list,
sccs: list,
@@ -570,7 +638,9 @@ def _select_everything(
# keeps them (``include_symbols``) since ``only_page_ids`` filters to the
# one the user asked for rather than emitting all of them.
symbol_spotlights=[t for _, t in symbols] if include_symbols else [],
- module_groups=[m for _, m in modules],
+ module_groups=[m for _, m in concepts.scored],
+ concept_groups=list(concepts.groups),
+ layer_labels=dict(concepts.layer_labels),
api_contract_paths=[p for _, p in apis],
infra_paths=[p for _, p in infras],
scc_groups=[g for _, g in sccs],
@@ -606,7 +676,8 @@ def select_pages(inputs: SelectionInputs) -> Selection:
# Build scored candidates for every bucket.
files = _build_file_candidates(inputs)
symbols = _build_symbol_candidates(inputs)
- modules = _build_module_groups(inputs)
+ concepts = _build_module_groups(inputs)
+ modules = concepts.scored
apis = _build_api_candidates(inputs)
infras = _build_infra_candidates(inputs)
sccs = _build_scc_candidates(inputs)
@@ -628,7 +699,7 @@ def select_pages(inputs: SelectionInputs) -> Selection:
# budget has no job to do. Take every candidate in every bucket. The
# tail stays empty because the file bucket already holds every file.
return _select_everything(
- files, symbols, modules, apis, infras, sccs, available, include_symbols=False
+ files, symbols, concepts, apis, infras, sccs, available, include_symbols=False
)
# Scoped generation (``repowise generate``): full-coverage allow-set so
@@ -636,7 +707,7 @@ def select_pages(inputs: SelectionInputs) -> Selection:
# regenerated on demand.
if getattr(inputs, "select_all", False):
return _select_everything(
- files, symbols, modules, apis, infras, sccs, available, include_symbols=True
+ files, symbols, concepts, apis, infras, sccs, available, include_symbols=True
)
allocation = allocate_budget(
@@ -687,6 +758,8 @@ def select_pages(inputs: SelectionInputs) -> Selection:
# budget: the grouper's size ladder holds the page count near 50-80 for
# a repository this size and scales sublinearly above it.
module_groups=[m for _, m in modules],
+ concept_groups=list(concepts.groups),
+ layer_labels=dict(concepts.layer_labels),
api_contract_paths=[p for _, p in apis[: allocation.api_contract]],
infra_paths=[p for _, p in infras[: allocation.infra_page]],
scc_groups=[g for _, g in sccs[: allocation.scc_page]],
diff --git a/tests/unit/generation/test_concept_naming_wiring.py b/tests/unit/generation/test_concept_naming_wiring.py
new file mode 100644
index 000000000..5572236a8
--- /dev/null
+++ b/tests/unit/generation/test_concept_naming_wiring.py
@@ -0,0 +1,426 @@
+"""The namer is wired into the real pipeline, and cannot break it.
+
+The naming machinery existed and was tested before any of this ran in
+production: the selector derived every shipped title from a path and the
+planner was reachable only from its own unit tests. So these tests assert
+reachability through ``run_pipeline`` rather than by calling the planner,
+because calling the planner is exactly what passed while nothing shipped.
+
+The rest assert the guarantees naming is allowed to make. It may change what
+a page is called. It may not change which files a page covers, what its id
+is, or whether the run finishes.
+"""
+
+from __future__ import annotations
+
+import json
+
+from repowise.core.generation.concept_tree.grouping import ConceptGroup
+from repowise.core.generation.concept_tree.planner import PlannerInputs, name_groups
+from repowise.core.pipeline import run_pipeline
+from repowise.core.pipeline.modes import OrchestratorMode
+from repowise.core.providers.llm.base import GeneratedResponse
+from repowise.core.providers.llm.mock import MockProvider
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+class NamingProvider(MockProvider):
+ """A provider that answers the naming call and mocks everything else.
+
+ Keyed off the naming system prompt rather than call order: naming happens
+ before any page is written, but pinning the assertion to "call zero" would
+ make an unrelated reordering look like a naming failure.
+ """
+
+ def __init__(self, naming_payload: object, **kwargs) -> None:
+ super().__init__(**kwargs)
+ self.naming_payload = naming_payload
+ self.naming_calls = 0
+
+ async def generate(self, system_prompt: str = "", user_prompt: str = "", **kwargs):
+ if "information architect" in system_prompt:
+ self.naming_calls += 1
+ self._calls.append({"system_prompt": system_prompt, **kwargs})
+ body = (
+ self.naming_payload
+ if isinstance(self.naming_payload, str)
+ else json.dumps(self.naming_payload)
+ )
+ return GeneratedResponse(content=body, input_tokens=10, output_tokens=10)
+ return await super().generate(system_prompt, user_prompt, **kwargs)
+
+
+#: Enough distinct subtrees, and enough files in each, that the grouper's size
+#: ladder produces several groups. A single-group fixture cannot test naming,
+#: ordering or title collisions, and it passed as though it could.
+_PACKAGES = ("ingest", "render", "storage", "transport", "analysis", "reporting")
+
+
+def _write_repo(repo_path):
+ """A repository with several clearly different subsystems and a root file."""
+ repo_path.mkdir(parents=True, exist_ok=True)
+ for pkg in _PACKAGES:
+ (repo_path / "src" / pkg).mkdir(parents=True, exist_ok=True)
+ for i in range(12):
+ (repo_path / "src" / pkg / f"mod{i}.py").write_text(
+ f"def {pkg}_{i}() -> int:\n return {i}\n", encoding="utf-8"
+ )
+ (repo_path / "main.py").write_text("def main() -> None:\n pass\n", encoding="utf-8")
+ return repo_path
+
+
+async def _run_deterministic(repo_path):
+ """The keyless run. A ``MockProvider`` is a provider, so it is not this."""
+ return await run_pipeline(
+ repo_path,
+ generate_docs=True,
+ llm_client=None,
+ concurrency=1,
+ mode=OrchestratorMode.DETERMINISTIC,
+ )
+
+
+def _naming_payload_for(titles: dict[str, str]) -> dict:
+ return {
+ "sections": [{"title": "Core Engine", "groups": list(titles)}],
+ "names": {
+ gid: {"title": title, "scope": f"Covers the {title} work and nothing else."}
+ for gid, title in titles.items()
+ },
+ }
+
+
+async def _run(repo_path, provider):
+ return await run_pipeline(
+ repo_path,
+ generate_docs=True,
+ llm_client=provider,
+ concurrency=1,
+ # No ``test_run``: it caps the pipeline to the top ten files by
+ # PageRank, which collapses this fixture to a single group, and a
+ # one-group run cannot exercise naming, ordering or collisions.
+ )
+
+
+def _module_pages(result):
+ return [p for p in result.generated_pages if p.page_type == "module_page"]
+
+
+# ---------------------------------------------------------------------------
+# Reachability
+# ---------------------------------------------------------------------------
+
+
+async def test_the_namer_runs_on_the_real_generation_path(tmp_path):
+ """A model title reaches a shipped page. This is the whole change."""
+ repo = _write_repo(tmp_path / "repo")
+ provider = NamingProvider(
+ _naming_payload_for({f"g{i:02d}": f"Title {i}" for i in range(1, 21)})
+ )
+
+ result = await _run(repo, provider)
+
+ assert provider.naming_calls == 1, "naming must be exactly one call, not one per page"
+ pages = _module_pages(result)
+ assert pages, "fixture produced no concept pages, so it cannot test naming"
+ assert any(
+ p.title.startswith("Title ") for p in pages
+ ), f"no model title reached a page: {[p.title for p in pages]}"
+
+
+async def test_the_keyless_path_never_calls_a_namer(tmp_path):
+ """Keyless is a real tree with plainer names, and it costs nothing.
+
+ Asserted through the stamp rather than through a spy: the keyless run has
+ no provider to record calls on, which is the point of it.
+ """
+ repo = _write_repo(tmp_path / "repo")
+
+ result = await _run_deterministic(repo)
+ pages = _module_pages(result)
+
+ assert pages
+ assert all(p.title.strip() for p in pages)
+ assert all(
+ "concept_order" not in p.metadata for p in pages
+ ), "a keyless run stamped a reading order, so something named it"
+
+
+async def test_a_run_that_writes_no_concept_page_does_not_name_one(tmp_path):
+ """An incremental update must not buy an outline it throws away.
+
+ ``file_pages_only`` returns before the concept level, so a naming call on
+ that path is a whole-repository request whose result nothing reads. On a
+ post-commit update hook that is a bill per commit for nothing.
+ """
+ from repowise.core.generation import GenerationConfig
+
+ repo = _write_repo(tmp_path / "repo")
+ provider = NamingProvider(_naming_payload_for({"g01": "Anything At All"}))
+
+ await run_pipeline(
+ repo,
+ generate_docs=True,
+ llm_client=provider,
+ concurrency=1,
+ generation_config=GenerationConfig(max_concurrency=1, file_pages_only=True),
+ )
+
+ assert provider.naming_calls == 0, "an incremental run paid for a naming call"
+
+
+# ---------------------------------------------------------------------------
+# Identity does not follow the title (D2)
+# ---------------------------------------------------------------------------
+
+
+async def test_renaming_a_page_does_not_mint_a_new_page(tmp_path):
+ """Two runs, different titles, identical ids and identical membership.
+
+ The load-bearing one. If identity followed the title, every regeneration
+ would strand the old row and a re-index would double the wiki.
+ """
+ repo = _write_repo(tmp_path / "repo")
+
+ first = await _run(
+ repo, NamingProvider(_naming_payload_for({f"g{i:02d}": f"Alpha {i}" for i in range(1, 21)}))
+ )
+ second = await _run(
+ repo, NamingProvider(_naming_payload_for({f"g{i:02d}": f"Omega {i}" for i in range(1, 21)}))
+ )
+
+ a = {p.page_id: p for p in _module_pages(first)}
+ b = {p.page_id: p for p in _module_pages(second)}
+
+ assert a and b
+ assert set(a) == set(b), "a rename moved a page id"
+ assert {p.structural_key for p in a.values()} == {p.structural_key for p in b.values()}
+ for pid, page in a.items():
+ assert page.metadata["file_paths"] == b[pid].metadata["file_paths"]
+ # The fixture is only meaningful if the titles actually differed.
+ assert {p.title for p in a.values()} != {p.title for p in b.values()}
+
+
+async def test_the_partition_is_identical_with_and_without_a_namer(tmp_path):
+ """D5: adding a key changes the prose, never the shape."""
+ repo = _write_repo(tmp_path / "repo")
+
+ named = await _run(
+ repo, NamingProvider(_naming_payload_for({f"g{i:02d}": f"Named {i}" for i in range(1, 21)}))
+ )
+ plain = await _run_deterministic(repo)
+
+ n = {p.page_id: sorted(p.metadata["file_paths"]) for p in _module_pages(named)}
+ p = {p.page_id: sorted(p.metadata["file_paths"]) for p in _module_pages(plain)}
+
+ assert n, "fixture produced no concept pages"
+ assert n == p
+
+
+# ---------------------------------------------------------------------------
+# A bad naming call costs titles, never the run
+# ---------------------------------------------------------------------------
+
+
+async def test_a_provider_that_raises_leaves_a_complete_wiki(tmp_path):
+ class Exploding(NamingProvider):
+ async def generate(self, system_prompt: str = "", user_prompt: str = "", **kwargs):
+ if "information architect" in system_prompt:
+ raise RuntimeError("naming is down")
+ return await MockProvider.generate(self, system_prompt, user_prompt, **kwargs)
+
+ repo = _write_repo(tmp_path / "repo")
+ result = await _run(repo, Exploding(None))
+
+ pages = _module_pages(result)
+ assert pages
+ assert all(p.title for p in pages)
+
+
+async def test_a_malformed_response_leaves_a_complete_wiki(tmp_path):
+ repo = _write_repo(tmp_path / "repo")
+ result = await _run(repo, NamingProvider("not json at all {{{"))
+
+ pages = _module_pages(result)
+ assert pages
+ assert all(p.title for p in pages)
+
+
+async def test_invented_group_ids_are_discarded(tmp_path):
+ """A response naming only groups that do not exist changes nothing."""
+ repo = _write_repo(tmp_path / "repo")
+ payload = _naming_payload_for({"g99": "Invented", "zzz": "Also Invented"})
+ result = await _run(repo, NamingProvider(payload))
+
+ pages = _module_pages(result)
+ assert pages
+ assert not any(p.title in ("Invented", "Also Invented") for p in pages)
+
+
+async def test_a_skipped_group_keeps_a_deterministic_title(tmp_path):
+ """Naming one of eight groups leaves the other seven named, not blank."""
+ repo = _write_repo(tmp_path / "repo")
+ result = await _run(repo, NamingProvider(_naming_payload_for({"g01": "Only This One"})))
+
+ pages = _module_pages(result)
+ assert len(pages) > 1, "fixture must produce more than one group"
+ assert all(p.title.strip() for p in pages)
+ assert len({p.title for p in pages}) == len(pages), "titles collided"
+
+
+# ---------------------------------------------------------------------------
+# The reasoning setting is honoured
+# ---------------------------------------------------------------------------
+
+
+async def test_the_naming_call_honours_the_run_reasoning_setting(tmp_path):
+ repo = _write_repo(tmp_path / "repo")
+ (repo / ".repowise").mkdir(exist_ok=True)
+ (repo / ".repowise" / "config.yaml").write_text("reasoning: off\n", encoding="utf-8")
+
+ provider = NamingProvider(_naming_payload_for({"g01": "Anything At All"}))
+ await _run(repo, provider)
+
+ naming = [
+ c for c in provider.calls if "information architect" in (c.get("system_prompt") or "")
+ ]
+ assert naming, "the naming call did not happen"
+ assert all(c.get("reasoning") == "off" for c in naming)
+
+
+# ---------------------------------------------------------------------------
+# Section and order
+# ---------------------------------------------------------------------------
+
+
+async def test_the_section_and_order_reach_the_page(tmp_path):
+ repo = _write_repo(tmp_path / "repo")
+ payload = {
+ "sections": [
+ {"title": "Second Section", "groups": ["g02"]},
+ {"title": "First Section", "groups": ["g01"]},
+ ],
+ "names": {
+ "g01": {"title": "Ingest Pipeline", "scope": "Covers ingest and not rendering."},
+ "g02": {"title": "Render Pipeline", "scope": "Covers rendering and not ingest."},
+ },
+ }
+ result = await _run(repo, NamingProvider(payload))
+
+ stamped = [p for p in _module_pages(result) if p.metadata.get("concept_section")]
+ assert stamped, "no page carried the namer's section"
+ orders = [p.metadata["concept_order"] for p in stamped]
+ assert len(set(orders)) == len(orders), "two pages share a position in the reading order"
+
+
+# ---------------------------------------------------------------------------
+# name_groups itself
+# ---------------------------------------------------------------------------
+
+
+def _groups() -> list[ConceptGroup]:
+ return [
+ ConceptGroup(
+ members=[f"src/a/f{i}.py" for i in range(3)], dirs=["src/a"], target_path="src/a"
+ ),
+ ConceptGroup(
+ members=[f"src/b/f{i}.py" for i in range(3)], dirs=["src/b"], target_path="src/b"
+ ),
+ ]
+
+
+async def test_name_groups_never_repartitions_what_it_is_given():
+ groups = _groups()
+ keys_before = [g.structural_key for g in groups]
+
+ outline, _ = await name_groups(
+ groups,
+ PlannerInputs(repo_name="r", production_files=[m for g in groups for m in g.members]),
+ provider=None,
+ )
+
+ assert [p.structural_key for p in outline.pages] == keys_before
+ assert [g.structural_key for g in groups] == keys_before, "naming mutated the grouping"
+
+
+async def test_repair_is_budgeted_for_the_number_of_titles_it_must_fix():
+ """A first call that names nothing leaves repair naming the whole repo.
+
+ Repair was written for the few titles a long request loses near the end of
+ its list, and a flat token budget covers that. It does not cover the case
+ that matters: every group falling back at once, where a truncated repair
+ ships an outline whose every title came from a path. Observed for real.
+ """
+ seen: list[int] = []
+
+ class SectionsOnly(MockProvider):
+ """Answers with sections and no names, so every group falls back."""
+
+ async def generate(self, system_prompt: str = "", user_prompt: str = "", **kwargs):
+ seen.append(int(kwargs.get("max_tokens") or 0))
+ if "GROUPS TO RENAME" in user_prompt:
+ return GeneratedResponse(content="{}", input_tokens=1, output_tokens=1)
+ gids = [f"g{i:02d}" for i in range(1, len(_many_groups()) + 1)]
+ return GeneratedResponse(
+ content=json.dumps({"sections": [{"title": "Core", "groups": gids}], "names": {}}),
+ input_tokens=1,
+ output_tokens=1,
+ )
+
+ groups = _many_groups()
+ await name_groups(
+ groups,
+ PlannerInputs(repo_name="r", production_files=[m for g in groups for m in g.members]),
+ provider=SectionsOnly(),
+ )
+
+ assert len(seen) == 2, "repair did not run after a response that named nothing"
+ assert seen[1] > 2000, (
+ f"repair asked for {seen[1]} tokens to rename {len(groups)} groups, "
+ "which truncates and leaves every title path-derived"
+ )
+
+
+def _many_groups() -> list[ConceptGroup]:
+ return [
+ ConceptGroup(
+ members=[f"src/p{n}/f{i}.py" for i in range(4)],
+ dirs=[f"src/p{n}"],
+ target_path=f"src/p{n}",
+ )
+ for n in range(30)
+ ]
+
+
+async def test_duplicate_model_titles_are_forced_apart():
+ """Repair can decline. A reader still must not see two identical rows."""
+
+ class SameTitle(MockProvider):
+ async def generate(self, system_prompt: str = "", user_prompt: str = "", **kwargs):
+ return GeneratedResponse(
+ content=json.dumps(
+ {
+ "sections": [{"title": "Core", "groups": ["g01", "g02"]}],
+ "names": {
+ "g01": {"title": "Shared Name", "scope": "Covers one half of it."},
+ "g02": {"title": "Shared Name", "scope": "Covers the other half."},
+ },
+ }
+ ),
+ input_tokens=1,
+ output_tokens=1,
+ )
+
+ groups = _groups()
+ outline, _ = await name_groups(
+ groups,
+ PlannerInputs(repo_name="r", production_files=[m for g in groups for m in g.members]),
+ provider=SameTitle(),
+ repair=False,
+ )
+
+ titles = [p.title for p in outline.pages]
+ assert len(set(titles)) == len(titles), f"duplicate titles survived: {titles}"
diff --git a/tests/unit/generation/test_page_tree.py b/tests/unit/generation/test_page_tree.py
index 5ff14d015..cdc5812c6 100644
--- a/tests/unit/generation/test_page_tree.py
+++ b/tests/unit/generation/test_page_tree.py
@@ -270,3 +270,73 @@ class TestDanglingParents:
for p in pages
if p.parent_page_id is not None and p.parent_page_id not in known
] == []
+
+
+class TestConceptReadingOrder:
+ """Concept pages sort by the order the namer chose, not by the alphabet.
+
+ Sibling order is what ``display_order`` and ``section_number`` are computed
+ from, so this is the whole reader-visible payoff of naming the tree. It
+ also has to survive the store: every ``update`` rebuilds the tree from
+ ``TreeNode``s rehydrated out of ``metadata_json`` rather than from the
+ pages a run just produced.
+ """
+
+ def _concepts(self, *specs):
+ """Concept pages under one layer. Path order is deliberately reversed."""
+ pages = [_page("repo_overview", "demo"), _page("layer_page", "layer:service")]
+ for target, order in specs:
+ meta = {"file_paths": [f"{target}/a.py"], "layer_id": "layer:service"}
+ if order is not None:
+ meta["concept_order"] = order
+ pages.append(_page("module_page", target, **meta))
+ return pages
+
+ def _ordered(self, pages):
+ concepts = [p for p in pages if p.page_type == "module_page"]
+ return [p.target_path for p in sorted(concepts, key=lambda p: p.display_order)]
+
+ def test_the_namers_order_wins_over_the_path(self):
+ # src/aaa sorts first alphabetically and is meant to be read last.
+ pages = self._concepts(("src/aaa", 2), ("src/mmm", 1), ("src/zzz", 0))
+ assign_page_tree(pages, LAYER_ORDER)
+
+ assert self._ordered(pages) == ["src/zzz", "src/mmm", "src/aaa"]
+ # The fixture is only meaningful if it disagrees with path order.
+ assert self._ordered(pages) != sorted(p.target_path for p in pages if p.page_type == "module_page")
+
+ def test_pages_written_before_naming_keep_their_path_order(self):
+ """A wiki indexed before this existed carries no order at all."""
+ pages = self._concepts(("src/aaa", None), ("src/mmm", None), ("src/zzz", None))
+ assign_page_tree(pages, LAYER_ORDER)
+
+ assert self._ordered(pages) == ["src/aaa", "src/mmm", "src/zzz"]
+
+ def test_an_unnamed_page_sorts_after_every_named_one(self):
+ """Position zero is a real place. An unnamed page must not borrow it."""
+ pages = self._concepts(("src/aaa", None), ("src/mmm", 1), ("src/zzz", 0))
+ assign_page_tree(pages, LAYER_ORDER)
+
+ assert self._ordered(pages) == ["src/zzz", "src/mmm", "src/aaa"]
+
+ def test_the_order_survives_the_store_round_trip(self):
+ """``update`` rebuilds from JSON metadata, not from generated pages."""
+ import json
+
+ from repowise.core.generation.page_tree import TreeNode
+
+ pages = self._concepts(("src/aaa", 2), ("src/mmm", 1), ("src/zzz", 0))
+ nodes = [
+ TreeNode(
+ page_id=p.page_id,
+ page_type=p.page_type,
+ target_path=p.target_path,
+ # Exactly what page_tree_sync does: store as JSON, read it back.
+ metadata=json.loads(json.dumps(p.metadata)),
+ )
+ for p in pages
+ ]
+ assign_page_tree(nodes, LAYER_ORDER)
+
+ assert self._ordered(nodes) == ["src/zzz", "src/mmm", "src/aaa"]
+ assert [n.section_number for n in nodes if n.page_type == "module_page"]
diff --git a/tests/unit/generation/test_selection_concept_groups.py b/tests/unit/generation/test_selection_concept_groups.py
index b1c197c0c..b2bf84176 100644
--- a/tests/unit/generation/test_selection_concept_groups.py
+++ b/tests/unit/generation/test_selection_concept_groups.py
@@ -96,7 +96,7 @@ def _inputs(
def test_every_production_file_is_covered_exactly_once():
- groups = [g for _, g in _build_module_groups(_inputs())]
+ groups = [g for _, g in _build_module_groups(_inputs()).scored]
claimed = [p for g in groups for p in g.file_paths]
assert sorted(claimed) == sorted(_paths()), "the partition is not total"
@@ -111,7 +111,7 @@ def test_root_level_files_get_a_usable_target_path():
and leaves the page with nothing for the tree, the breadcrumbs or the
bench gold set to match on.
"""
- groups = [g for _, g in _build_module_groups(_inputs())]
+ groups = [g for _, g in _build_module_groups(_inputs()).scored]
owning = [g for g in groups if "setup.py" in g.file_paths]
assert owning, "root-level files were dropped from the partition"
@@ -125,7 +125,7 @@ def test_root_level_files_get_a_usable_target_path():
def test_test_files_never_enter_the_concept_tree():
"""D8: tests keep file pages, they do not get concept pages."""
- groups = [g for _, g in _build_module_groups(_inputs())]
+ groups = [g for _, g in _build_module_groups(_inputs()).scored]
claimed = {p for g in groups for p in g.file_paths}
assert not any(p.startswith("tests/") for p in claimed)
@@ -135,8 +135,8 @@ def test_test_files_never_enter_the_concept_tree():
def test_dropping_the_test_files_does_not_change_the_partition():
"""Excluding tests must be a filter, not a force that reshapes groups."""
- with_tests = [g.structural_key for _, g in _build_module_groups(_inputs())]
- without = [g.structural_key for _, g in _build_module_groups(_inputs(with_tests=False))]
+ with_tests = [g.structural_key for _, g in _build_module_groups(_inputs()).scored]
+ without = [g.structural_key for _, g in _build_module_groups(_inputs(with_tests=False)).scored]
assert with_tests == without
@@ -148,7 +148,7 @@ def test_dropping_the_test_files_does_not_change_the_partition():
def test_groups_carry_a_concept_prefixed_structural_key():
"""The prefix is what lets a wiki tell a concept page from an old one."""
- groups = [g for _, g in _build_module_groups(_inputs())]
+ groups = [g for _, g in _build_module_groups(_inputs()).scored]
assert groups
assert all(g.structural_key.startswith("concept-") for g in groups)
@@ -162,7 +162,7 @@ def test_target_paths_are_unique():
Two groups resolving to one target is a row collision: the second page
silently overwrites the first on persist.
"""
- groups = [g for _, g in _build_module_groups(_inputs())]
+ groups = [g for _, g in _build_module_groups(_inputs()).scored]
keys = [g.key for g in groups]
assert len(keys) == len(set(keys)), f"duplicate target paths: {keys}"
@@ -180,7 +180,7 @@ def test_identity_is_stable_across_processes():
from tests.unit.generation.test_selection_concept_groups import (
_build_module_groups, _inputs,
)
- groups = [g for _, g in _build_module_groups(_inputs())]
+ groups = [g for _, g in _build_module_groups(_inputs()).scored]
print("|".join(f"{g.key}={g.structural_key}" for g in groups))
"""
)
@@ -202,6 +202,84 @@ def test_identity_is_stable_across_processes():
assert outs[0], "the subprocess produced no groups, so this asserted nothing"
+# ---------------------------------------------------------------------------
+# The importance floor covers documentation and example source too
+# ---------------------------------------------------------------------------
+
+
+def _support_paths() -> list[str]:
+ """Documentation and example source at the repository root."""
+ return (
+ [f"docs/conf{i}.py" for i in range(4)]
+ + [f"docs_src/tutorial/app{i}.py" for i in range(8)]
+ + [f"examples/basic/demo{i}.py" for i in range(5)]
+ + ["samples/quickstart.py"]
+ )
+
+
+def test_root_documentation_and_examples_get_no_concept_page():
+ """Measured, not assumed: on one framework these outnumbered the library."""
+ groups = [g for _, g in _build_module_groups(_inputs(paths=_paths() + _support_paths())).scored]
+ claimed = [p for g in groups for p in g.file_paths]
+
+ assert claimed, "fixture produced no groups"
+ for prefix in ("docs/", "docs_src/", "examples/", "samples/"):
+ assert not any(
+ p.startswith(prefix) for p in claimed
+ ), f"{prefix} reached the concept tree: {[p for p in claimed if p.startswith(prefix)]}"
+ # The fixture is only meaningful if those files were in the input.
+ assert any(p.startswith("docs_src/") for p in _support_paths())
+
+
+def test_a_docs_directory_inside_a_package_keeps_its_pages():
+ """The rule is anchored at the repository root, so a docs *feature* stays.
+
+ This is the case the amendment names: a source directory that happens to
+ be called ``docs`` deep inside a package is code, and excluding it would
+ be the substring bug the test rule originally shipped with, wearing a
+ different hat.
+ """
+ feature = [f"packages/app/src/core/docs/render{i}.py" for i in range(7)]
+ groups = [g for _, g in _build_module_groups(_inputs(paths=_paths() + feature)).scored]
+ claimed = {p for g in groups for p in g.file_paths}
+
+ assert set(feature) <= claimed, f"a docs feature was excluded: {set(feature) - claimed}"
+
+
+def test_production_paths_that_merely_begin_with_the_word_are_kept():
+ """Segment match, not prefix-of-string match."""
+ lookalikes = [
+ "documentation_builder/render.py",
+ "docsearch/index.py",
+ "examples_runner/main.py",
+ "sampler/draw.py",
+ ]
+ groups = [g for _, g in _build_module_groups(_inputs(paths=_paths() + lookalikes)).scored]
+ claimed = {p for g in groups for p in g.file_paths}
+
+ assert set(lookalikes) <= claimed, f"excluded by a prefix match: {set(lookalikes) - claimed}"
+
+
+def test_dropping_support_files_does_not_change_the_partition():
+ """The floor is a pure filter: it must not reshape the remaining groups."""
+ with_support = [
+ g.structural_key
+ for _, g in _build_module_groups(_inputs(paths=_paths() + _support_paths())).scored
+ ]
+ without = [g.structural_key for _, g in _build_module_groups(_inputs(paths=_paths())).scored]
+
+ assert with_support == without
+
+
+def test_a_repository_that_is_only_documentation_still_gets_a_tree():
+ """Documenting the docs is a bad wiki. Having none at all is a worse one."""
+ groups = [g for _, g in _build_module_groups(_inputs(paths=_support_paths())).scored]
+ claimed = [p for g in groups for p in g.file_paths]
+
+ assert groups, "the floor emptied the tree instead of yielding"
+ assert sorted(claimed) == sorted(_support_paths())
+
+
# ---------------------------------------------------------------------------
# Ordering and naming
# ---------------------------------------------------------------------------
@@ -209,7 +287,7 @@ def test_identity_is_stable_across_processes():
def test_ranked_by_summed_pagerank_not_by_path():
"""The heaviest subsystem sorts first even though its path sorts last."""
- scored = _build_module_groups(_inputs())
+ scored = _build_module_groups(_inputs()).scored
ordered_keys = [g.key for _, g in scored]
assert len(scored) > 1, "a one-group fixture cannot test ordering"
@@ -227,7 +305,7 @@ def test_ranked_by_summed_pagerank_not_by_path():
def test_display_is_a_name_not_a_bare_path():
- groups = [g for _, g in _build_module_groups(_inputs())]
+ groups = [g for _, g in _build_module_groups(_inputs()).scored]
for g in groups:
assert g.display, f"{g.key} has no title"
@@ -244,8 +322,8 @@ def test_layer_labels_from_the_kg_reach_the_title():
"nodeIds": [f"file:{p}" for p in _paths() if "/ui/c4/" in p],
}
]
- with_kg = [g for _, g in _build_module_groups(_inputs(kg_modules=modules))]
- without_kg = [g for _, g in _build_module_groups(_inputs())]
+ with_kg = [g for _, g in _build_module_groups(_inputs(kg_modules=modules)).scored]
+ without_kg = [g for _, g in _build_module_groups(_inputs()).scored]
# Membership is unchanged by the layer signal: it steers merging of
# adjacent runs, it never forces a split.
@@ -269,7 +347,7 @@ def test_layer_labels_from_the_kg_reach_the_title():
def test_select_pages_emits_every_group():
inputs = _inputs()
- groups = _build_module_groups(inputs)
+ groups = _build_module_groups(inputs).scored
selection = select_pages(inputs)
assert len(selection.module_groups) == len(groups)
@@ -302,7 +380,7 @@ def test_two_groups_never_share_a_title():
+ [f"packages/beta/src/ui/f{i}.py" for i in range(8)]
+ ["setup.py"]
)
- groups = [g for _, g in _build_module_groups(_inputs(paths=paths))]
+ groups = [g for _, g in _build_module_groups(_inputs(paths=paths)).scored]
titles = [g.display for g in groups]
assert len(titles) == len(set(titles)), titles