Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): a model names the concept pages, and the tree stops documenting documentation (#1025)
Raghav Chamadiyaf88039132c4e
diff --git a/packages/core/src/repowise/core/generation/concept_tree/planner.py b/packages/core/src/repowise/core/generation/concept_tree/planner.pyindex 73e29e3ee..6d13a1ef0 100644--- a/packages/core/src/repowise/core/generation/concept_tree/planner.py+++ b/packages/core/src/repowise/core/generation/concept_tree/planner.py@@ -33,6 +33,7 @@ from .naming import (decode_response,deterministic_scope,deterministic_title,+ disambiguate_titles,ground_scopes,parse_json_object,)@@ -71,6 +72,17 @@ class PlannerInputs:test_files: set[str] = field(default_factory=set)+def _reasoning_kwargs(reasoning: str | None) -> dict[str, str]:+ """Pass the run's reasoning setting through, or nothing when unset.++ A user who turned thinking off did so for the whole run, and a naming call+ that quietly re-enables it bills them for what they declined. Omitted+ rather than defaulted when unset so a provider that does not take the+ keyword is not handed one.+ """+ return {"reasoning": reasoning} if reasoning else {}++@contextlib.contextmanagerdef _billed_as(provider: Any, operation: str) -> Iterator[None]:"""Label this step's spend on the Costs page; no-op without a tracker."""@@ -154,24 +166,15 @@ def _disambiguate_titles(named: list[NamedGroup]) -> None:seen[entry.title.lower()] = 1-def plan_deterministic(- inputs: PlannerInputs, *, params: GroupingParams | None = None-) -> tuple[ConceptOutline, list[ConceptGroup]]:- """The keyless outline: real structure, names derived from paths and layers.+def name_deterministically(groups: list[ConceptGroup], inputs: PlannerInputs) -> ConceptOutline:+ """Title and section already-computed *groups* from paths and layers only.- Per D5 this is the same tree the LLM path produces — identical grouping,- identical identities — with plainer titles. Adding a key upgrades the- prose, never the shape, so a user who adds one later does not get their- wiki re-partitioned underneath them.-- That guarantee only holds if both paths partition with the same bounds, so- *params* is threaded rather than re-derived. It was not, once: the caller's- bounds were dropped here and the two paths produced different page counts- for the same repository.+ Split out from :func:`plan_deterministic` so a caller that has already+ partitioned the files can name that exact partition rather than asking for+ a second one. Two callers each running the grouper is two places that have+ to agree about page identity, which is the arrangement D2 exists to+ prevent."""- groups = group_files(- inputs.production_files, layer_of_file=inputs.layer_of_file, params=params- )named = [NamedGroup(group=g,@@ -187,7 +190,26 @@ def plan_deterministic(outline = _build_outline(named)_merge_thin_sections(outline)outline.naming_mode = "deterministic"- return outline, groups+ return outline+++def plan_deterministic(+ inputs: PlannerInputs, *, params: GroupingParams | None = None+) -> tuple[ConceptOutline, list[ConceptGroup]]:+ """The keyless outline: real structure, names derived from paths and layers.++ Per D5 this is the same tree the LLM path produces — identical grouping,+ identical identities — with plainer titles. Adding a key upgrades the+ prose, never the shape, so a user who adds one later does not get their+ wiki re-partitioned underneath them.++ That guarantee only holds if both paths partition with the same bounds, so+ *params* is threaded rather than re-derived. It was not, once: the caller's+ bounds were dropped here and the two paths produced different page counts+ for the same repository.+ """+ groups = group_files(inputs.production_files, layer_of_file=inputs.layer_of_file, params=params)+ return name_deterministically(groups, inputs), groups_REPAIR_INSTRUCTIONS = """\@@ -243,6 +265,20 @@ def _usable_scope(scope: str, target_path: str) -> bool:return "/" not in text.split(" ", 1)[0] or " " in text.strip()+def _force_unique_titles(outline: ConceptOutline) -> int:+ """Qualify any title still shared by two pages. Returns how many moved."""+ pages = outline.pages+ unique = disambiguate_titles([(p.title, p.target_path) for p in pages])+ changed = 0+ for page, title in zip(pages, unique, strict=True):+ if title != page.title:+ page.title = title+ changed += 1+ if changed:+ logger.info("concept_outline_titles_disambiguated", changed=changed)+ return changed++def _repair_targets(outline: ConceptOutline, report: OutlineReport) -> set[str]:"""The page titles worth a second call. Everything else is left alone.@@ -267,18 +303,46 @@ async def plan_outline(deterministic: bool = False,params: GroupingParams | None = None,repair: bool = True,+ reasoning: str | None = None,) -> tuple[ConceptOutline, OutlineReport]:- """Produce a validated outline for *inputs*.+ """Group *inputs* and produce a validated outline over that grouping."""+ groups = group_files(inputs.production_files, layer_of_file=inputs.layer_of_file, params=params)+ return await name_groups(+ groups,+ inputs,+ provider=provider,+ deterministic=deterministic,+ params=params,+ repair=repair,+ reasoning=reasoning,+ )- Falls back to :func:`plan_deterministic` whenever there is no provider, the- run is in deterministic mode, or the model's response cannot be used. A- failed naming call costs the wiki its titles, never its structure.++async def name_groups(+ groups: list[ConceptGroup],+ inputs: PlannerInputs,+ *,+ provider: Any | None = None,+ deterministic: bool = False,+ params: GroupingParams | None = None,+ repair: bool = True,+ reasoning: str | None = None,+) -> tuple[ConceptOutline, OutlineReport]:+ """Name and section an already-computed partition, then validate it.++ One call names every group. The model receives opaque ids and returns a+ title, a scope and a section per id; it never decides membership, so a+ response that is late, partial, malformed or full of invented ids costs+ the wiki its titles and never its structure or its coverage.++ Falls back to :func:`name_deterministically` whenever there is no provider,+ the run is in deterministic mode, or the model's response cannot be used."""all_files = set(inputs.production_files)resolved = params or params_for(len(all_files))if deterministic or provider is None:- outline, _groups = plan_deterministic(inputs, params=params)+ outline = name_deterministically(groups, inputs)report = validate_outline(outline,all_files=all_files,@@ -287,9 +351,6 @@ async def plan_outline()return outline, report- groups = group_files(- inputs.production_files, layer_of_file=inputs.layer_of_file, params=params- )payload, index = build_payload(groups,layer_labels=inputs.layer_labels,@@ -333,6 +394,7 @@ async def plan_outline(user_prompt=instructions + body,max_tokens=16000,temperature=0.2,+ **_reasoning_kwargs(reasoning),)data = parse_json_object(getattr(response, "content", "") or "")except Exception as exc:@@ -343,9 +405,7 @@ async def plan_outline(# and could take the run down — the one failure mode this design exists to# rule out. Decoding is defensive in its own right; this is the second line.try:- named, invented_ids = decode_response(- data, index, layer_labels=inputs.layer_labels- )+ named, invented_ids = decode_response(data, index, layer_labels=inputs.layer_labels)except Exception as exc:logger.warning("concept_outline_decode_failed", error=str(exc))named, invented_ids = decode_response({}, index, layer_labels=inputs.layer_labels)@@ -369,7 +429,7 @@ async def plan_outline(if repair:targets = _repair_targets(outline, report)if targets:- await _repair_titles(outline, index, targets, provider=provider)+ await _repair_titles(outline, index, targets, provider=provider, reasoning=reasoning)# Repair writes new prose, so it has to face the same citation# check the first pass did. Grounding after the last write rather# than after the first is the difference between checking the page@@ -383,6 +443,21 @@ async def plan_outline()report.invented_paths = sorted(set(report.invented_paths) | set(ungrounded))+ # Last line of defence on titles. Repair is a second model call and can+ # decline, run out of ids, or hand back a name already in use, so a+ # duplicate can still reach here. Identity is structural, so nothing is+ # lost when two titles collide, but a reader sees two identical rows and+ # cannot tell which is which. Qualifying them by path is the same rule the+ # deterministic path already applies, and it cannot fail.+ if _force_unique_titles(outline):+ report = validate_outline(+ outline,+ all_files=all_files,+ test_files=inputs.test_files,+ max_files_per_page=resolved.max_files,+ )+ report.invented_paths = sorted(set(report.invented_paths) | set(ungrounded))+logger.info("concept_outline_planned",pages=report.page_count,@@ -390,6 +465,11 @@ async def plan_outline(coverage=round(report.coverage, 4),invented=len(report.invented_paths),naming_mode=outline.naming_mode,+ # How many titles the model actually decided. ``naming_mode`` only says+ # that at least one did, so a run where the response was unusable and+ # every page fell back to its path still reads as "llm". That happened+ # and looked like success; this is the number that shows it.+ named_by_model=sum(1 for p in outline.pages if p.named_by_model),)return outline, report@@ -400,6 +480,7 @@ async def _repair_titles(targets: set[str],*,provider: Any,+ reasoning: str | None = None,) -> None:"""Re-ask for just the failing titles, then apply only what improved.@@ -436,13 +517,22 @@ async def _repair_titles(taken="\n".join(f"- {t}" for t in taken) or "(none)",failures="\n".join(lines),)+ # Budgeted per failing group rather than flat. Repair was written for the+ # handful a long request loses near the end of the list, and a flat 2000+ # tokens covers that. It does not cover the case that actually matters: a+ # first call that comes back with no usable names at all, where repair is+ # asked to name the whole repository and truncates, so an outline that+ # could have been recovered ships with every title derived from a path.+ # Observed on this repository at 82 groups.+ budget = max(2000, min(16000, 200 + 150 * len(failing)))try:with _billed_as(provider, COST_OPERATION):response = await provider.generate(system_prompt=SYSTEM_PROMPT,user_prompt=prompt,- max_tokens=2000,+ max_tokens=budget,temperature=0.2,+ **_reasoning_kwargs(reasoning),)data = parse_json_object(getattr(response, "content", "") or "")except Exception as exc:diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.pyindex c193e3662..aac397cce 100644--- a/packages/core/src/repowise/core/generation/page_generator/levels.py+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py@@ -311,6 +311,8 @@ def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:target_path=mg.key,structural_key=mg.structural_key,members=list(mg.file_paths),+ section=mg.section,+ order=mg.order,),))diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.pyindex 78657045d..fbfb2387b 100644--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py@@ -22,6 +22,7 @@ from ..context_assembler import FilePageContextfrom ..models import (STRUCTURALLY_KEYED_PAGE_TYPES,GeneratedPage,+ compute_page_id,member_structural_key,)from . import levels as _levels@@ -212,7 +213,7 @@ class _GenerationRun:already_completed=len(self.completed_ids),)- def _compute_selection(self) -> None:+ async def _compute_selection(self) -> None:"""Run the selection subsystem and derive the level allow-sets."""code_files = [p@@ -278,6 +279,14 @@ class _GenerationRun:self.sel_module_groups = list(selection.module_groups)self.sel_scc_groups = list(selection.scc_groups)+ # One model call names every concept group. It runs here, after the+ # partition exists and before any page id is computed, because naming+ # is the one part of the concept tree that is judgement rather than+ # structure: the grouper decides what a page covers, the model decides+ # what to call it. Kept out of ``select_pages`` so the cost estimator+ # and scope resolution, which call it too, stay free of the model.+ await self._name_concept_groups()+# Tiered doc generation: split the selected file pages into a full-LLM# tier-1 and a deterministic template-only tier-2. When tier1_top_n is# None this puts every selected page in tier-1 (no behaviour change).@@ -308,6 +317,121 @@ class _GenerationRun:),)+ async def _name_concept_groups(self) -> None:+ """Give every concept group a title, a scope and a section, in one call.++ The model receives opaque group ids and returns a name per id. It never+ sees the membership question, so the two failures the probes measured+ have no entry point here: a group it skips keeps the deterministic+ title selection already gave it, and an id it invents is discarded.++ Both the keyless and the named path read the *same* partition, because+ selection computed it once and this only relabels it. D5's guarantee+ that adding a key changes the prose and never the shape is therefore+ structural rather than a property two code paths have to agree on.++ Every failure mode degrades to those deterministic titles rather than+ failing the run: no provider, deterministic mode, a provider that+ raises, a response that will not parse, or a group the model left out.+ """+ groups = list(getattr(self.selection, "concept_groups", None) or [])+ deterministic = bool(getattr(self.config, "deterministic", False))+ provider = getattr(self.gen, "_provider", None)+ if not groups or deterministic or provider is None:+ return++ # Naming is only worth a call if this run is going to write a concept+ # page with the result. Three runs reach here and do not: an+ # incremental update sets ``file_pages_only`` and returns before level+ # 4, a scoped run may have asked for pages of other types only, and a+ # resumed run may already hold every concept page. Each would otherwise+ # buy a whole-repository outline and discard it, which on a post-commit+ # update hook is a bill per commit for nothing.+ if getattr(self.config, "file_pages_only", False):+ return+ if not any(self._emit(compute_page_id("module_page", mg.key)) for mg in self.sel_module_groups):+ return++ from ..concept_tree.planner import PlannerInputs, name_groups++ inputs = PlannerInputs(+ repo_name=self.repo_name or "",+ # The grouping is total, so the union of the members is exactly the+ # file set the grouper partitioned. Rebuilt from the groups rather+ # than re-filtered from ``parsed_files`` so the validator measures+ # coverage against the set that was actually grouped.+ production_files=[m for g in groups for m in g.members],+ repo_root=Path(self.repo_path) if self.repo_path else None,+ layer_labels=dict(getattr(self.selection, "layer_labels", None) or {}),+ entry_points={+ p.file_info.path+ for p in self.parsed_files+ if getattr(p.file_info, "is_entry_point", False)+ },+ )++ try:+ outline, report = await name_groups(+ groups,+ inputs,+ provider=provider,+ reasoning=getattr(self.config, "reasoning", None),+ )+ except Exception as exc:+ # ``name_groups`` guards the call and the decode itself, so reaching+ # here means a defect rather than a bad response. The titles from+ # selection are already correct, so the wiki is worse-named and not+ # broken, and that is the trade this whole step is allowed to make.+ log.warning("concept_naming.failed", error=str(exc))+ return++ title_of: dict[str, str] = {}+ section_of: dict[str, str] = {}+ order_of: dict[str, int] = {}+ position = 0+ for section in outline.sections:+ for page in section.pages:+ title_of[page.structural_key] = page.title+ section_of[page.structural_key] = section.title+ order_of[page.structural_key] = position+ position += 1++ from dataclasses import replace as _replace++ renamed = 0+ groups_out = []+ for mg in self.sel_module_groups:+ title = title_of.get(mg.structural_key)+ if title is None:+ # The outline covers every group it was given, so this is a+ # group that never reached the namer. It keeps what it had.+ groups_out.append(mg)+ continue+ if title != mg.display:+ renamed += 1+ groups_out.append(+ _replace(+ mg,+ display=title,+ section=section_of.get(mg.structural_key, ""),+ order=order_of.get(mg.structural_key, 0),+ )+ )+ # Generation order is left alone: it is summed PageRank, so the most+ # central subsystem is written first and lands in the store earliest.+ # Where the reader meets each page is ``order``, applied by the tree.+ self.sel_module_groups = groups_out++ log.info(+ "concept_naming.applied",+ groups=len(groups),+ renamed=renamed,+ sections=len(outline.sections),+ naming_mode=outline.naming_mode,+ duplicate_titles=len(report.duplicate_titles),+ invented=len(report.invented_paths),+ )+def _compute_demand(self) -> dict[str, int]:"""Per-file question demand for the FAQ-weighted budget tilt.@@ -574,7 +698,7 @@ class _GenerationRun:async def execute(self) -> list[GeneratedPage]:self._setup_job()await self._seed_resume()- self._compute_selection()+ await self._compute_selection()self._announce_total()self._compute_ia()diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.pyindex 6d2b1b1a1..ee3d4ac11 100644--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py@@ -110,6 +110,8 @@ class PerTypeGenerationMixin:target_path: str | None = None,structural_key: str = "",members: list[str] | None = None,+ section: str = "",+ order: int = 0,) -> GeneratedPage:ctx = self._assembler.assemble_module_page(title,@@ -164,11 +166,30 @@ class PerTypeGenerationMixin:# dropped files would be parented somewhere else entirely, and the two# records of what the page covers would disagree.covered = sorted(members) if members else sorted(fc.file_path for fc in file_contexts)- if self._config.deterministic:- page = self._deterministic_module_page(ctx, page_target, title, module_git_summary)++ def _stamp_concept(page: GeneratedPage) -> GeneratedPage:+ """Record where the namer put this page in the reading order.++ Only stamped when a namer ran, so an absent key means "nobody+ decided", which is what the tree needs to tell apart from "first".+ Display only: the tree reads ``concept_order`` to order siblings,+ and neither field takes part in page identity, so re-sectioning a+ wiki renumbers it and mints no new pages.+ """page.metadata["file_paths"] = covered+ if section:+ page.metadata["concept_section"] = section+ page.metadata["concept_order"] = order+ # Set by the producer, preserved by ``_stamp_structural_keys``. The+ # grouper hashed exactly this member list when it decided what the+ # group was, so recomputing it downstream would give two places+ # that must agree about page identity — the arrangement D2 rules out.page.structural_key = structural_key or page.structural_keyreturn page++ if self._config.deterministic:+ page = self._deterministic_module_page(ctx, page_target, title, module_git_summary)+ return _stamp_concept(page)user_prompt = self._render("module_page.j2", ctx=ctx, module_git_summary=module_git_summary)response = await self._call_provider("module_page", user_prompt, str(uuid.uuid4()), target_path=page_target@@ -181,13 +202,7 @@ class PerTypeGenerationMixin:compute_source_hash(user_prompt),GENERATION_LEVELS["module_page"],)- page.metadata["file_paths"] = covered- # Set by the producer, preserved by ``_stamp_structural_keys``. The- # grouper hashed exactly this member list when it decided what the- # group was, so recomputing it downstream would give two places that- # must agree about page identity — the arrangement D2 rules out.- page.structural_key = structural_key or page.structural_key- return page+ return _stamp_concept(page)async def generate_scc_page(self,diff --git a/packages/core/src/repowise/core/generation/page_tree.py b/packages/core/src/repowise/core/generation/page_tree.pyindex 45eac9956..a184ec5c5 100644--- a/packages/core/src/repowise/core/generation/page_tree.py+++ b/packages/core/src/repowise/core/generation/page_tree.py@@ -102,6 +102,8 @@ def _sort_key(page: GeneratedPage, layer_rank: dict[str, int]) -> tuple:rank = _onboarding_rank(page)elif page.page_type == "layer_page":rank = layer_rank.get(page.target_path, len(layer_rank))+ elif page.page_type == "module_page":+ rank = _concept_rank(page)else:rank = 0return (@@ -112,6 +114,29 @@ def _sort_key(page: GeneratedPage, layer_rank: dict[str, int]) -> tuple:)+def _concept_rank(page: GeneratedPage) -> int:+ """Where the namer put this concept page in the reading order.++ A concept page's siblings are the other concept pages under the same+ layer, and sorting them by target path puts the reader wherever the+ alphabet lands. The namer already decided an order as a narrative, so use+ it when it exists.++ Pages with no order come from a run that never named them (keyless, or a+ wiki written before naming shipped) and sort after the named ones, where+ the tie-break on ``target_path`` gives them the order they have today.+ Deliberately not zero: that is a real position, and borrowing it would+ interleave unnamed pages through the narrative.+ """+ order = page.metadata.get("concept_order")+ return order if isinstance(order, int) else _UNRANKED_CONCEPT+++# Larger than any real position, so unnamed pages sort after named ones for+# any outline the grouper's size ladder can produce.+_UNRANKED_CONCEPT = 1_000_000++def _dominant_layer(paths: Iterable[str], layer_of_file: dict[str, str]) -> str:"""The layer most of these files belong to.diff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.pyindex a854fd56f..cf8d0a60f 100644--- a/packages/core/src/repowise/core/generation/selection/selector.py+++ b/packages/core/src/repowise/core/generation/selection/selector.py@@ -17,7 +17,7 @@ import structlogfrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY-from ..concept_tree.grouping import group_files+from ..concept_tree.grouping import ConceptGroup, group_filesfrom ..concept_tree.naming import deterministic_title, disambiguate_titlesfrom ..models import scc_page_slugfrom ..tour import DEFAULT_MAX_LANDMARKS, tour_landmark_paths@@ -42,6 +42,13 @@ _INFRA_LANGUAGES = _LANG_REGISTRY.infra_languages()_INFRA_FILENAMES = frozenset({"Dockerfile", "Makefile", "GNUmakefile"})_CODE_LANGUAGES = _LANG_REGISTRY.code_languages()+# Top-level directories whose contents document or illustrate the repository+# rather than being it. Matched as a whole first path segment only. See+# ``_is_support_file`` for why the anchoring is the point.+_SUPPORT_ROOT_DIRS = frozenset(+ {"docs", "doc", "documentation", "docs_src", "examples", "example", "samples", "sample"}+)+# ---------------------------------------------------------------------------# Output dataclasses@@ -72,6 +79,26 @@ class ModuleGroup:label: str | None = Nonecohesion: float | None = Nonestructural_key: str = ""+ #: The section this page belongs to, and its position in the reading order+ #: the namer chose. Display only: neither takes part in page identity, so a+ #: re-run that re-sections the wiki renumbers it and mints nothing.+ section: str = ""+ order: int = 0+++@dataclass+class ConceptCandidates:+ """The concept partition, in both the shapes the run needs.++ ``scored`` is what the budget and the generator consume. ``groups`` is the+ partition itself, carried out so the namer can title the exact groups that+ were computed here instead of asking the grouper for a second partition+ and hoping the two agree.+ """++ scored: list[tuple[float, ModuleGroup]] = field(default_factory=list)+ groups: list[ConceptGroup] = field(default_factory=list)+ layer_labels: dict[str, str] = field(default_factory=dict)@dataclass@@ -93,6 +120,12 @@ class Selection:# (Phase G coverage tail). Kept separate from ``file_page_paths`` so cost# estimation stays honest (these are free) and the CLI can report the split.deterministic_tail_paths: list[str] = field(default_factory=list)+ # The concept partition behind ``module_groups``, one entry per group, plus+ # the layer names the grouper steered by. Carried so a caller holding a+ # provider can name these groups; selection itself stays free of the model+ # so the cost estimator and scope resolution keep costing nothing.+ concept_groups: list[ConceptGroup] = field(default_factory=list)+ layer_labels: dict[str, str] = field(default_factory=dict)def counts(self) -> dict[str, int]:"""Per-page-type counts of BUDGETED (LLM-costed) pages.@@ -305,7 +338,28 @@ def _layer_map_from_kg(return layer_of_file, labels-def _build_module_groups(inputs: SelectionInputs) -> list[tuple[float, ModuleGroup]]:+def _is_support_file(path: str) -> bool:+ """Whether *path* is documentation or example source rather than the subject.++ Anchored at the repository root and matched on whole path segments, which+ is the difference between this rule and the substring match the test rule+ originally shipped with. A package that has a ``docs`` directory as a+ *feature* keeps its pages; a ``docs/`` tree at the top of the repository+ does not, because it is the subject's documentation and not the subject.++ Measured rather than assumed, the same way the test exclusion was: on one+ web framework 10 of 15 concept pages documented the example snippets that+ illustrate the library while the library itself got 4. These files match a+ concept query on vocabulary without answering how the system works, and a+ wiki that documents its subject's documentation has said nothing about the+ subject. They keep their deterministic file pages, so nothing becomes+ unreachable.+ """+ head = path.split("/", 1)[0].lower()+ return head in _SUPPORT_ROOT_DIRS+++def _build_module_groups(inputs: SelectionInputs) -> ConceptCandidates:"""Return the concept groups, scored, one per ``module_page``.This is the concept tree, not one page per directory. The partition comes@@ -326,13 +380,27 @@ def _build_module_groups(inputs: SelectionInputs) -> list[tuple[float, ModuleGroscore no longer rations anything — see :func:`select_pages` — but pageorder is still what the reader sees first."""- files = [+ production = [p.file_info.pathfor p in inputs.parsed_filesif _is_code_file(p) and not getattr(p.file_info, "is_test", False)]+ files = [p for p in production if not _is_support_file(p)]if not files:- return []+ # A repository whose production code is entirely under one of those+ # roots. Documenting the docs is a bad wiki; having no wiki at all is+ # a worse one, so the floor yields rather than emptying the tree.+ if production:+ log.info("page_selection.support_floor_skipped", files=len(production))+ files = production+ elif len(files) != len(production):+ log.info(+ "page_selection.support_floor",+ excluded=len(production) - len(files),+ kept=len(files),+ )+ if not files:+ return ConceptCandidates()layer_of_file, layer_labels = _layer_map_from_kg(inputs)groups = group_files(files, layer_of_file=layer_of_file)@@ -372,7 +440,7 @@ def _build_module_groups(inputs: SelectionInputs) -> list[tuple[float, ModuleGro))scored.sort(key=lambda x: (-x[0], x[1].key))- return scored+ return ConceptCandidates(scored=scored, groups=groups, layer_labels=layer_labels)def _build_api_candidates(inputs: SelectionInputs) -> list[tuple[float, str]]:@@ -537,7 +605,7 @@ def _ensure_landmarks(selected: list[str], landmarks: list[str]) -> list[str]:def _select_everything(files: list,symbols: list,- modules: list,+ concepts: ConceptCandidates,apis: list,infras: list,sccs: list,@@ -570,7 +638,9 @@ def _select_everything(# keeps them (``include_symbols``) since ``only_page_ids`` filters to the# one the user asked for rather than emitting all of them.symbol_spotlights=[t for _, t in symbols] if include_symbols else [],- module_groups=[m for _, m in modules],+ module_groups=[m for _, m in concepts.scored],+ concept_groups=list(concepts.groups),+ layer_labels=dict(concepts.layer_labels),api_contract_paths=[p for _, p in apis],infra_paths=[p for _, p in infras],scc_groups=[g for _, g in sccs],@@ -606,7 +676,8 @@ def select_pages(inputs: SelectionInputs) -> Selection:# Build scored candidates for every bucket.files = _build_file_candidates(inputs)symbols = _build_symbol_candidates(inputs)- modules = _build_module_groups(inputs)+ concepts = _build_module_groups(inputs)+ modules = concepts.scoredapis = _build_api_candidates(inputs)infras = _build_infra_candidates(inputs)sccs = _build_scc_candidates(inputs)@@ -628,7 +699,7 @@ def select_pages(inputs: SelectionInputs) -> Selection:# budget has no job to do. Take every candidate in every bucket. The# tail stays empty because the file bucket already holds every file.return _select_everything(- files, symbols, modules, apis, infras, sccs, available, include_symbols=False+ files, symbols, concepts, apis, infras, sccs, available, include_symbols=False)# Scoped generation (``repowise generate``): full-coverage allow-set so@@ -636,7 +707,7 @@ def select_pages(inputs: SelectionInputs) -> Selection:# regenerated on demand.if getattr(inputs, "select_all", False):return _select_everything(- files, symbols, modules, apis, infras, sccs, available, include_symbols=True+ files, symbols, concepts, apis, infras, sccs, available, include_symbols=True)allocation = allocate_budget(@@ -687,6 +758,8 @@ def select_pages(inputs: SelectionInputs) -> Selection:# budget: the grouper's size ladder holds the page count near 50-80 for# a repository this size and scales sublinearly above it.module_groups=[m for _, m in modules],+ concept_groups=list(concepts.groups),+ layer_labels=dict(concepts.layer_labels),api_contract_paths=[p for _, p in apis[: allocation.api_contract]],infra_paths=[p for _, p in infras[: allocation.infra_page]],scc_groups=[g for _, g in sccs[: allocation.scc_page]],diff --git a/tests/unit/generation/test_concept_naming_wiring.py b/tests/unit/generation/test_concept_naming_wiring.pynew file mode 100644index 000000000..5572236a8--- /dev/null+++ b/tests/unit/generation/test_concept_naming_wiring.py@@ -0,0 +1,426 @@+"""The namer is wired into the real pipeline, and cannot break it.++The naming machinery existed and was tested before any of this ran in+production: the selector derived every shipped title from a path and the+planner was reachable only from its own unit tests. So these tests assert+reachability through ``run_pipeline`` rather than by calling the planner,+because calling the planner is exactly what passed while nothing shipped.++The rest assert the guarantees naming is allowed to make. It may change what+a page is called. It may not change which files a page covers, what its id+is, or whether the run finishes.+"""++from __future__ import annotations++import json++from repowise.core.generation.concept_tree.grouping import ConceptGroup+from repowise.core.generation.concept_tree.planner import PlannerInputs, name_groups+from repowise.core.pipeline import run_pipeline+from repowise.core.pipeline.modes import OrchestratorMode+from repowise.core.providers.llm.base import GeneratedResponse+from repowise.core.providers.llm.mock import MockProvider++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++class NamingProvider(MockProvider):+ """A provider that answers the naming call and mocks everything else.++ Keyed off the naming system prompt rather than call order: naming happens+ before any page is written, but pinning the assertion to "call zero" would+ make an unrelated reordering look like a naming failure.+ """++ def __init__(self, naming_payload: object, **kwargs) -> None:+ super().__init__(**kwargs)+ self.naming_payload = naming_payload+ self.naming_calls = 0++ async def generate(self, system_prompt: str = "", user_prompt: str = "", **kwargs):+ if "information architect" in system_prompt:+ self.naming_calls += 1+ self._calls.append({"system_prompt": system_prompt, **kwargs})+ body = (+ self.naming_payload+ if isinstance(self.naming_payload, str)+ else json.dumps(self.naming_payload)+ )+ return GeneratedResponse(content=body, input_tokens=10, output_tokens=10)+ return await super().generate(system_prompt, user_prompt, **kwargs)+++#: Enough distinct subtrees, and enough files in each, that the grouper's size+#: ladder produces several groups. A single-group fixture cannot test naming,+#: ordering or title collisions, and it passed as though it could.+_PACKAGES = ("ingest", "render", "storage", "transport", "analysis", "reporting")+++def _write_repo(repo_path):+ """A repository with several clearly different subsystems and a root file."""+ repo_path.mkdir(parents=True, exist_ok=True)+ for pkg in _PACKAGES:+ (repo_path / "src" / pkg).mkdir(parents=True, exist_ok=True)+ for i in range(12):+ (repo_path / "src" / pkg / f"mod{i}.py").write_text(+ f"def {pkg}_{i}() -> int:\n return {i}\n", encoding="utf-8"+ )+ (repo_path / "main.py").write_text("def main() -> None:\n pass\n", encoding="utf-8")+ return repo_path+++async def _run_deterministic(repo_path):+ """The keyless run. A ``MockProvider`` is a provider, so it is not this."""+ return await run_pipeline(+ repo_path,+ generate_docs=True,+ llm_client=None,+ concurrency=1,+ mode=OrchestratorMode.DETERMINISTIC,+ )+++def _naming_payload_for(titles: dict[str, str]) -> dict:+ return {+ "sections": [{"title": "Core Engine", "groups": list(titles)}],+ "names": {+ gid: {"title": title, "scope": f"Covers the {title} work and nothing else."}+ for gid, title in titles.items()+ },+ }+++async def _run(repo_path, provider):+ return await run_pipeline(+ repo_path,+ generate_docs=True,+ llm_client=provider,+ concurrency=1,+ # No ``test_run``: it caps the pipeline to the top ten files by+ # PageRank, which collapses this fixture to a single group, and a+ # one-group run cannot exercise naming, ordering or collisions.+ )+++def _module_pages(result):+ return [p for p in result.generated_pages if p.page_type == "module_page"]+++# ---------------------------------------------------------------------------+# Reachability+# ---------------------------------------------------------------------------+++async def test_the_namer_runs_on_the_real_generation_path(tmp_path):+ """A model title reaches a shipped page. This is the whole change."""+ repo = _write_repo(tmp_path / "repo")+ provider = NamingProvider(+ _naming_payload_for({f"g{i:02d}": f"Title {i}" for i in range(1, 21)})+ )++ result = await _run(repo, provider)++ assert provider.naming_calls == 1, "naming must be exactly one call, not one per page"+ pages = _module_pages(result)+ assert pages, "fixture produced no concept pages, so it cannot test naming"+ assert any(+ p.title.startswith("Title ") for p in pages+ ), f"no model title reached a page: {[p.title for p in pages]}"+++async def test_the_keyless_path_never_calls_a_namer(tmp_path):+ """Keyless is a real tree with plainer names, and it costs nothing.++ Asserted through the stamp rather than through a spy: the keyless run has+ no provider to record calls on, which is the point of it.+ """+ repo = _write_repo(tmp_path / "repo")++ result = await _run_deterministic(repo)+ pages = _module_pages(result)++ assert pages+ assert all(p.title.strip() for p in pages)+ assert all(+ "concept_order" not in p.metadata for p in pages+ ), "a keyless run stamped a reading order, so something named it"+++async def test_a_run_that_writes_no_concept_page_does_not_name_one(tmp_path):+ """An incremental update must not buy an outline it throws away.++ ``file_pages_only`` returns before the concept level, so a naming call on+ that path is a whole-repository request whose result nothing reads. On a+ post-commit update hook that is a bill per commit for nothing.+ """+ from repowise.core.generation import GenerationConfig++ repo = _write_repo(tmp_path / "repo")+ provider = NamingProvider(_naming_payload_for({"g01": "Anything At All"}))++ await run_pipeline(+ repo,+ generate_docs=True,+ llm_client=provider,+ concurrency=1,+ generation_config=GenerationConfig(max_concurrency=1, file_pages_only=True),+ )++ assert provider.naming_calls == 0, "an incremental run paid for a naming call"+++# ---------------------------------------------------------------------------+# Identity does not follow the title (D2)+# ---------------------------------------------------------------------------+++async def test_renaming_a_page_does_not_mint_a_new_page(tmp_path):+ """Two runs, different titles, identical ids and identical membership.++ The load-bearing one. If identity followed the title, every regeneration+ would strand the old row and a re-index would double the wiki.+ """+ repo = _write_repo(tmp_path / "repo")++ first = await _run(+ repo, NamingProvider(_naming_payload_for({f"g{i:02d}": f"Alpha {i}" for i in range(1, 21)}))+ )+ second = await _run(+ repo, NamingProvider(_naming_payload_for({f"g{i:02d}": f"Omega {i}" for i in range(1, 21)}))+ )++ a = {p.page_id: p for p in _module_pages(first)}+ b = {p.page_id: p for p in _module_pages(second)}++ assert a and b+ assert set(a) == set(b), "a rename moved a page id"+ assert {p.structural_key for p in a.values()} == {p.structural_key for p in b.values()}+ for pid, page in a.items():+ assert page.metadata["file_paths"] == b[pid].metadata["file_paths"]+ # The fixture is only meaningful if the titles actually differed.+ assert {p.title for p in a.values()} != {p.title for p in b.values()}+++async def test_the_partition_is_identical_with_and_without_a_namer(tmp_path):+ """D5: adding a key changes the prose, never the shape."""+ repo = _write_repo(tmp_path / "repo")++ named = await _run(+ repo, NamingProvider(_naming_payload_for({f"g{i:02d}": f"Named {i}" for i in range(1, 21)}))+ )+ plain = await _run_deterministic(repo)++ n = {p.page_id: sorted(p.metadata["file_paths"]) for p in _module_pages(named)}+ p = {p.page_id: sorted(p.metadata["file_paths"]) for p in _module_pages(plain)}++ assert n, "fixture produced no concept pages"+ assert n == p+++# ---------------------------------------------------------------------------+# A bad naming call costs titles, never the run+# ---------------------------------------------------------------------------+++async def test_a_provider_that_raises_leaves_a_complete_wiki(tmp_path):+ class Exploding(NamingProvider):+ async def generate(self, system_prompt: str = "", user_prompt: str = "", **kwargs):+ if "information architect" in system_prompt:+ raise RuntimeError("naming is down")+ return await MockProvider.generate(self, system_prompt, user_prompt, **kwargs)++ repo = _write_repo(tmp_path / "repo")+ result = await _run(repo, Exploding(None))++ pages = _module_pages(result)+ assert pages+ assert all(p.title for p in pages)+++async def test_a_malformed_response_leaves_a_complete_wiki(tmp_path):+ repo = _write_repo(tmp_path / "repo")+ result = await _run(repo, NamingProvider("not json at all {{{"))++ pages = _module_pages(result)+ assert pages+ assert all(p.title for p in pages)+++async def test_invented_group_ids_are_discarded(tmp_path):+ """A response naming only groups that do not exist changes nothing."""+ repo = _write_repo(tmp_path / "repo")+ payload = _naming_payload_for({"g99": "Invented", "zzz": "Also Invented"})+ result = await _run(repo, NamingProvider(payload))++ pages = _module_pages(result)+ assert pages+ assert not any(p.title in ("Invented", "Also Invented") for p in pages)+++async def test_a_skipped_group_keeps_a_deterministic_title(tmp_path):+ """Naming one of eight groups leaves the other seven named, not blank."""+ repo = _write_repo(tmp_path / "repo")+ result = await _run(repo, NamingProvider(_naming_payload_for({"g01": "Only This One"})))++ pages = _module_pages(result)+ assert len(pages) > 1, "fixture must produce more than one group"+ assert all(p.title.strip() for p in pages)+ assert len({p.title for p in pages}) == len(pages), "titles collided"+++# ---------------------------------------------------------------------------+# The reasoning setting is honoured+# ---------------------------------------------------------------------------+++async def test_the_naming_call_honours_the_run_reasoning_setting(tmp_path):+ repo = _write_repo(tmp_path / "repo")+ (repo / ".repowise").mkdir(exist_ok=True)+ (repo / ".repowise" / "config.yaml").write_text("reasoning: off\n", encoding="utf-8")++ provider = NamingProvider(_naming_payload_for({"g01": "Anything At All"}))+ await _run(repo, provider)++ naming = [+ c for c in provider.calls if "information architect" in (c.get("system_prompt") or "")+ ]+ assert naming, "the naming call did not happen"+ assert all(c.get("reasoning") == "off" for c in naming)+++# ---------------------------------------------------------------------------+# Section and order+# ---------------------------------------------------------------------------+++async def test_the_section_and_order_reach_the_page(tmp_path):+ repo = _write_repo(tmp_path / "repo")+ payload = {+ "sections": [+ {"title": "Second Section", "groups": ["g02"]},+ {"title": "First Section", "groups": ["g01"]},+ ],+ "names": {+ "g01": {"title": "Ingest Pipeline", "scope": "Covers ingest and not rendering."},+ "g02": {"title": "Render Pipeline", "scope": "Covers rendering and not ingest."},+ },+ }+ result = await _run(repo, NamingProvider(payload))++ stamped = [p for p in _module_pages(result) if p.metadata.get("concept_section")]+ assert stamped, "no page carried the namer's section"+ orders = [p.metadata["concept_order"] for p in stamped]+ assert len(set(orders)) == len(orders), "two pages share a position in the reading order"+++# ---------------------------------------------------------------------------+# name_groups itself+# ---------------------------------------------------------------------------+++def _groups() -> list[ConceptGroup]:+ return [+ ConceptGroup(+ members=[f"src/a/f{i}.py" for i in range(3)], dirs=["src/a"], target_path="src/a"+ ),+ ConceptGroup(+ members=[f"src/b/f{i}.py" for i in range(3)], dirs=["src/b"], target_path="src/b"+ ),+ ]+++async def test_name_groups_never_repartitions_what_it_is_given():+ groups = _groups()+ keys_before = [g.structural_key for g in groups]++ outline, _ = await name_groups(+ groups,+ PlannerInputs(repo_name="r", production_files=[m for g in groups for m in g.members]),+ provider=None,+ )++ assert [p.structural_key for p in outline.pages] == keys_before+ assert [g.structural_key for g in groups] == keys_before, "naming mutated the grouping"+++async def test_repair_is_budgeted_for_the_number_of_titles_it_must_fix():+ """A first call that names nothing leaves repair naming the whole repo.++ Repair was written for the few titles a long request loses near the end of+ its list, and a flat token budget covers that. It does not cover the case+ that matters: every group falling back at once, where a truncated repair+ ships an outline whose every title came from a path. Observed for real.+ """+ seen: list[int] = []++ class SectionsOnly(MockProvider):+ """Answers with sections and no names, so every group falls back."""++ async def generate(self, system_prompt: str = "", user_prompt: str = "", **kwargs):+ seen.append(int(kwargs.get("max_tokens") or 0))+ if "GROUPS TO RENAME" in user_prompt:+ return GeneratedResponse(content="{}", input_tokens=1, output_tokens=1)+ gids = [f"g{i:02d}" for i in range(1, len(_many_groups()) + 1)]+ return GeneratedResponse(+ content=json.dumps({"sections": [{"title": "Core", "groups": gids}], "names": {}}),+ input_tokens=1,+ output_tokens=1,+ )++ groups = _many_groups()+ await name_groups(+ groups,+ PlannerInputs(repo_name="r", production_files=[m for g in groups for m in g.members]),+ provider=SectionsOnly(),+ )++ assert len(seen) == 2, "repair did not run after a response that named nothing"+ assert seen[1] > 2000, (+ f"repair asked for {seen[1]} tokens to rename {len(groups)} groups, "+ "which truncates and leaves every title path-derived"+ )+++def _many_groups() -> list[ConceptGroup]:+ return [+ ConceptGroup(+ members=[f"src/p{n}/f{i}.py" for i in range(4)],+ dirs=[f"src/p{n}"],+ target_path=f"src/p{n}",+ )+ for n in range(30)+ ]+++async def test_duplicate_model_titles_are_forced_apart():+ """Repair can decline. A reader still must not see two identical rows."""++ class SameTitle(MockProvider):+ async def generate(self, system_prompt: str = "", user_prompt: str = "", **kwargs):+ return GeneratedResponse(+ content=json.dumps(+ {+ "sections": [{"title": "Core", "groups": ["g01", "g02"]}],+ "names": {+ "g01": {"title": "Shared Name", "scope": "Covers one half of it."},+ "g02": {"title": "Shared Name", "scope": "Covers the other half."},+ },+ }+ ),+ input_tokens=1,+ output_tokens=1,+ )++ groups = _groups()+ outline, _ = await name_groups(+ groups,+ PlannerInputs(repo_name="r", production_files=[m for g in groups for m in g.members]),+ provider=SameTitle(),+ repair=False,+ )++ titles = [p.title for p in outline.pages]+ assert len(set(titles)) == len(titles), f"duplicate titles survived: {titles}"diff --git a/tests/unit/generation/test_page_tree.py b/tests/unit/generation/test_page_tree.pyindex 5ff14d015..cdc5812c6 100644--- a/tests/unit/generation/test_page_tree.py+++ b/tests/unit/generation/test_page_tree.py@@ -270,3 +270,73 @@ class TestDanglingParents:for p in pagesif p.parent_page_id is not None and p.parent_page_id not in known] == []+++class TestConceptReadingOrder:+ """Concept pages sort by the order the namer chose, not by the alphabet.++ Sibling order is what ``display_order`` and ``section_number`` are computed+ from, so this is the whole reader-visible payoff of naming the tree. It+ also has to survive the store: every ``update`` rebuilds the tree from+ ``TreeNode``s rehydrated out of ``metadata_json`` rather than from the+ pages a run just produced.+ """++ def _concepts(self, *specs):+ """Concept pages under one layer. Path order is deliberately reversed."""+ pages = [_page("repo_overview", "demo"), _page("layer_page", "layer:service")]+ for target, order in specs:+ meta = {"file_paths": [f"{target}/a.py"], "layer_id": "layer:service"}+ if order is not None:+ meta["concept_order"] = order+ pages.append(_page("module_page", target, **meta))+ return pages++ def _ordered(self, pages):+ concepts = [p for p in pages if p.page_type == "module_page"]+ return [p.target_path for p in sorted(concepts, key=lambda p: p.display_order)]++ def test_the_namers_order_wins_over_the_path(self):+ # src/aaa sorts first alphabetically and is meant to be read last.+ pages = self._concepts(("src/aaa", 2), ("src/mmm", 1), ("src/zzz", 0))+ assign_page_tree(pages, LAYER_ORDER)++ assert self._ordered(pages) == ["src/zzz", "src/mmm", "src/aaa"]+ # The fixture is only meaningful if it disagrees with path order.+ assert self._ordered(pages) != sorted(p.target_path for p in pages if p.page_type == "module_page")++ def test_pages_written_before_naming_keep_their_path_order(self):+ """A wiki indexed before this existed carries no order at all."""+ pages = self._concepts(("src/aaa", None), ("src/mmm", None), ("src/zzz", None))+ assign_page_tree(pages, LAYER_ORDER)++ assert self._ordered(pages) == ["src/aaa", "src/mmm", "src/zzz"]++ def test_an_unnamed_page_sorts_after_every_named_one(self):+ """Position zero is a real place. An unnamed page must not borrow it."""+ pages = self._concepts(("src/aaa", None), ("src/mmm", 1), ("src/zzz", 0))+ assign_page_tree(pages, LAYER_ORDER)++ assert self._ordered(pages) == ["src/zzz", "src/mmm", "src/aaa"]++ def test_the_order_survives_the_store_round_trip(self):+ """``update`` rebuilds from JSON metadata, not from generated pages."""+ import json++ from repowise.core.generation.page_tree import TreeNode++ pages = self._concepts(("src/aaa", 2), ("src/mmm", 1), ("src/zzz", 0))+ nodes = [+ TreeNode(+ page_id=p.page_id,+ page_type=p.page_type,+ target_path=p.target_path,+ # Exactly what page_tree_sync does: store as JSON, read it back.+ metadata=json.loads(json.dumps(p.metadata)),+ )+ for p in pages+ ]+ assign_page_tree(nodes, LAYER_ORDER)++ assert self._ordered(nodes) == ["src/zzz", "src/mmm", "src/aaa"]+ assert [n.section_number for n in nodes if n.page_type == "module_page"]diff --git a/tests/unit/generation/test_selection_concept_groups.py b/tests/unit/generation/test_selection_concept_groups.pyindex b1c197c0c..b2bf84176 100644--- a/tests/unit/generation/test_selection_concept_groups.py+++ b/tests/unit/generation/test_selection_concept_groups.py@@ -96,7 +96,7 @@ def _inputs(def test_every_production_file_is_covered_exactly_once():- groups = [g for _, g in _build_module_groups(_inputs())]+ groups = [g for _, g in _build_module_groups(_inputs()).scored]claimed = [p for g in groups for p in g.file_paths]assert sorted(claimed) == sorted(_paths()), "the partition is not total"@@ -111,7 +111,7 @@ def test_root_level_files_get_a_usable_target_path():and leaves the page with nothing for the tree, the breadcrumbs or thebench gold set to match on."""- groups = [g for _, g in _build_module_groups(_inputs())]+ groups = [g for _, g in _build_module_groups(_inputs()).scored]owning = [g for g in groups if "setup.py" in g.file_paths]assert owning, "root-level files were dropped from the partition"@@ -125,7 +125,7 @@ def test_root_level_files_get_a_usable_target_path():def test_test_files_never_enter_the_concept_tree():"""D8: tests keep file pages, they do not get concept pages."""- groups = [g for _, g in _build_module_groups(_inputs())]+ groups = [g for _, g in _build_module_groups(_inputs()).scored]claimed = {p for g in groups for p in g.file_paths}assert not any(p.startswith("tests/") for p in claimed)@@ -135,8 +135,8 @@ def test_test_files_never_enter_the_concept_tree():def test_dropping_the_test_files_does_not_change_the_partition():"""Excluding tests must be a filter, not a force that reshapes groups."""- with_tests = [g.structural_key for _, g in _build_module_groups(_inputs())]- without = [g.structural_key for _, g in _build_module_groups(_inputs(with_tests=False))]+ with_tests = [g.structural_key for _, g in _build_module_groups(_inputs()).scored]+ without = [g.structural_key for _, g in _build_module_groups(_inputs(with_tests=False)).scored]assert with_tests == without@@ -148,7 +148,7 @@ def test_dropping_the_test_files_does_not_change_the_partition():def test_groups_carry_a_concept_prefixed_structural_key():"""The prefix is what lets a wiki tell a concept page from an old one."""- groups = [g for _, g in _build_module_groups(_inputs())]+ groups = [g for _, g in _build_module_groups(_inputs()).scored]assert groupsassert all(g.structural_key.startswith("concept-") for g in groups)@@ -162,7 +162,7 @@ def test_target_paths_are_unique():Two groups resolving to one target is a row collision: the second pagesilently overwrites the first on persist."""- groups = [g for _, g in _build_module_groups(_inputs())]+ groups = [g for _, g in _build_module_groups(_inputs()).scored]keys = [g.key for g in groups]assert len(keys) == len(set(keys)), f"duplicate target paths: {keys}"@@ -180,7 +180,7 @@ def test_identity_is_stable_across_processes():from tests.unit.generation.test_selection_concept_groups import (_build_module_groups, _inputs,)- groups = [g for _, g in _build_module_groups(_inputs())]+ groups = [g for _, g in _build_module_groups(_inputs()).scored]print("|".join(f"{g.key}={g.structural_key}" for g in groups))""")@@ -202,6 +202,84 @@ def test_identity_is_stable_across_processes():assert outs[0], "the subprocess produced no groups, so this asserted nothing"+# ---------------------------------------------------------------------------+# The importance floor covers documentation and example source too+# ---------------------------------------------------------------------------+++def _support_paths() -> list[str]:+ """Documentation and example source at the repository root."""+ return (+ [f"docs/conf{i}.py" for i in range(4)]+ + [f"docs_src/tutorial/app{i}.py" for i in range(8)]+ + [f"examples/basic/demo{i}.py" for i in range(5)]+ + ["samples/quickstart.py"]+ )+++def test_root_documentation_and_examples_get_no_concept_page():+ """Measured, not assumed: on one framework these outnumbered the library."""+ groups = [g for _, g in _build_module_groups(_inputs(paths=_paths() + _support_paths())).scored]+ claimed = [p for g in groups for p in g.file_paths]++ assert claimed, "fixture produced no groups"+ for prefix in ("docs/", "docs_src/", "examples/", "samples/"):+ assert not any(+ p.startswith(prefix) for p in claimed+ ), f"{prefix} reached the concept tree: {[p for p in claimed if p.startswith(prefix)]}"+ # The fixture is only meaningful if those files were in the input.+ assert any(p.startswith("docs_src/") for p in _support_paths())+++def test_a_docs_directory_inside_a_package_keeps_its_pages():+ """The rule is anchored at the repository root, so a docs *feature* stays.++ This is the case the amendment names: a source directory that happens to+ be called ``docs`` deep inside a package is code, and excluding it would+ be the substring bug the test rule originally shipped with, wearing a+ different hat.+ """+ feature = [f"packages/app/src/core/docs/render{i}.py" for i in range(7)]+ groups = [g for _, g in _build_module_groups(_inputs(paths=_paths() + feature)).scored]+ claimed = {p for g in groups for p in g.file_paths}++ assert set(feature) <= claimed, f"a docs feature was excluded: {set(feature) - claimed}"+++def test_production_paths_that_merely_begin_with_the_word_are_kept():+ """Segment match, not prefix-of-string match."""+ lookalikes = [+ "documentation_builder/render.py",+ "docsearch/index.py",+ "examples_runner/main.py",+ "sampler/draw.py",+ ]+ groups = [g for _, g in _build_module_groups(_inputs(paths=_paths() + lookalikes)).scored]+ claimed = {p for g in groups for p in g.file_paths}++ assert set(lookalikes) <= claimed, f"excluded by a prefix match: {set(lookalikes) - claimed}"+++def test_dropping_support_files_does_not_change_the_partition():+ """The floor is a pure filter: it must not reshape the remaining groups."""+ with_support = [+ g.structural_key+ for _, g in _build_module_groups(_inputs(paths=_paths() + _support_paths())).scored+ ]+ without = [g.structural_key for _, g in _build_module_groups(_inputs(paths=_paths())).scored]++ assert with_support == without+++def test_a_repository_that_is_only_documentation_still_gets_a_tree():+ """Documenting the docs is a bad wiki. Having none at all is a worse one."""+ groups = [g for _, g in _build_module_groups(_inputs(paths=_support_paths())).scored]+ claimed = [p for g in groups for p in g.file_paths]++ assert groups, "the floor emptied the tree instead of yielding"+ assert sorted(claimed) == sorted(_support_paths())++# ---------------------------------------------------------------------------# Ordering and naming# ---------------------------------------------------------------------------@@ -209,7 +287,7 @@ def test_identity_is_stable_across_processes():def test_ranked_by_summed_pagerank_not_by_path():"""The heaviest subsystem sorts first even though its path sorts last."""- scored = _build_module_groups(_inputs())+ scored = _build_module_groups(_inputs()).scoredordered_keys = [g.key for _, g in scored]assert len(scored) > 1, "a one-group fixture cannot test ordering"@@ -227,7 +305,7 @@ def test_ranked_by_summed_pagerank_not_by_path():def test_display_is_a_name_not_a_bare_path():- groups = [g for _, g in _build_module_groups(_inputs())]+ groups = [g for _, g in _build_module_groups(_inputs()).scored]for g in groups:assert g.display, f"{g.key} has no title"@@ -244,8 +322,8 @@ def test_layer_labels_from_the_kg_reach_the_title():"nodeIds": [f"file:{p}" for p in _paths() if "/ui/c4/" in p],}]- with_kg = [g for _, g in _build_module_groups(_inputs(kg_modules=modules))]- without_kg = [g for _, g in _build_module_groups(_inputs())]+ with_kg = [g for _, g in _build_module_groups(_inputs(kg_modules=modules)).scored]+ without_kg = [g for _, g in _build_module_groups(_inputs()).scored]# Membership is unchanged by the layer signal: it steers merging of# adjacent runs, it never forces a split.@@ -269,7 +347,7 @@ def test_layer_labels_from_the_kg_reach_the_title():def test_select_pages_emits_every_group():inputs = _inputs()- groups = _build_module_groups(inputs)+ groups = _build_module_groups(inputs).scoredselection = select_pages(inputs)assert len(selection.module_groups) == len(groups)@@ -302,7 +380,7 @@ def test_two_groups_never_share_a_title():+ [f"packages/beta/src/ui/f{i}.py" for i in range(8)]+ ["setup.py"])- groups = [g for _, g in _build_module_groups(_inputs(paths=paths))]+ groups = [g for _, g in _build_module_groups(_inputs(paths=paths)).scored]titles = [g.display for g in groups]assert len(titles) == len(set(titles)), titles