Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): concept groups replace per-directory module pages (#1024)
Raghav Chamadiyaabdc3a3fe3f9
diff --git a/packages/core/src/repowise/core/cost_estimator/approx.py b/packages/core/src/repowise/core/cost_estimator/approx.pyindex 9706a7627..2e38f8f97 100644--- a/packages/core/src/repowise/core/cost_estimator/approx.py+++ b/packages/core/src/repowise/core/cost_estimator/approx.py@@ -16,12 +16,6 @@ from __future__ import annotationsfrom .types import PageTypePlan-# Observed module-page ratio: curated/community grouping lands near one-# module page per ~25 selected file pages, floored at 1 and capped so a-# monorepo estimate does not balloon.-_FILES_PER_MODULE = 25-_MAX_MODULE_PAGES = 24-# The curated onboarding collection has 8 fixed slots; slots without# enough signal are skipped, so this biases slightly high on tiny repos._ONBOARDING_SLOTS = 8@@ -38,9 +32,27 @@ def approximate_generation_plan(file_pages = round(file_count * max(0.0, min(coverage_pct, 1.0)))if file_count > 0:file_pages = max(1, file_pages)- module_pages = (- min(_MAX_MODULE_PAGES, max(1, file_pages // _FILES_PER_MODULE)) if file_pages else 0- )+ # Concept pages partition every production file, so they scale with the+ # repository rather than with the budgeted file-page slice, and they are+ # not capped: a cap would quote a number the run cannot honour. The size+ # ladder the grouper actually uses gives the expected group size, so this+ # estimate moves with the real bounds instead of a remembered ratio.+ #+ # The ladder is a step function and this is fed *documentable* files+ # (tests and config included) while the grouper sees production code+ # only, so near a band edge the two can sit in different bands. Taking+ # the larger of this band and the one below keeps the error on the+ # over-quote side, which is the only side that does not surprise someone+ # with a bill.+ module_pages = 0+ if file_count:+ from repowise.core.generation.concept_tree.grouping import params_for++ estimates = [+ max(1, round(file_count / ((bounds.min_files + bounds.max_files) / 2)))+ for bounds in (params_for(file_count), params_for(int(file_count * 0.8)))+ ]+ module_pages = max(estimates)plans = [PageTypePlan(page_type="file_page", count=file_pages, level=0),diff --git a/packages/core/src/repowise/core/cost_estimator/plans.py b/packages/core/src/repowise/core/cost_estimator/plans.pyindex f6afc65b0..d3421e4e8 100644--- a/packages/core/src/repowise/core/cost_estimator/plans.py+++ b/packages/core/src/repowise/core/cost_estimator/plans.py@@ -37,10 +37,10 @@ def build_generation_plan(config:GenerationConfig — its ``coverage_pct`` drives the budget.kg_modules:- Curated wiki modules from the KG artifact, when available. Must be- passed for ``module_grouping="curated"`` estimates to match the- actual run — without it the selector falls back to community- grouping, exactly as generation itself would without an artifact.+ Curated wiki modules from the KG artifact, when available. Only+ steers which adjacent directories merge into a concept group, so its+ absence changes which pages exist rather than how many, and the+ estimate holds either way."""files = parsed_filesif skip_tests:diff --git a/packages/core/src/repowise/core/generation/concept_tree/grouping.py b/packages/core/src/repowise/core/generation/concept_tree/grouping.pyindex d6c0b967e..8a5ba8ba0 100644--- a/packages/core/src/repowise/core/generation/concept_tree/grouping.py+++ b/packages/core/src/repowise/core/generation/concept_tree/grouping.py@@ -116,9 +116,7 @@ class ConceptGroup:def __post_init__(self) -> None:if not self.structural_key:- self.structural_key = member_structural_key(- self.members, prefix=STRUCTURAL_KEY_PREFIX- )+ self.structural_key = member_structural_key(self.members, prefix=STRUCTURAL_KEY_PREFIX)@propertydef file_count(self) -> int:@@ -309,9 +307,7 @@ class _Partitioner:return []-def _absorb_thin(- groups: list[ConceptGroup], part: _Partitioner-) -> list[ConceptGroup]:+def _absorb_thin(groups: list[ConceptGroup], part: _Partitioner) -> list[ConceptGroup]:"""Fold groups too small to deserve a page into their closest neighbour.The recursion flushes a remainder wherever a subtree boundary falls, so a@@ -382,6 +378,18 @@ def _assign_targets(groups: list[ConceptGroup]) -> None:wrong is two pages silently sharing a row, so the loop below checks theresult and walks to the next candidate rather than trusting it."""+ # The repository root's directory path is the empty string, which is a+ # real directory but cannot be a page id: the id is ``"{type}:{path}"``,+ # so an empty path mints ``module_page:`` and leaves the page with nothing+ # for the tree, the breadcrumbs or a gold set to match on. It gets a name+ # here, inside the function that owns uniqueness, rather than downstream —+ # a caller remapping ``""`` to ``"root"`` afterwards would collide with a+ # repository that has a top-level directory actually called ``root``.+ all_dirs = {d for group in groups for d in group.dirs}+ root_name = "root"+ while root_name in all_dirs:+ root_name = f"_{root_name}"+taken: set[str] = set()# Largest first so the group with the strongest claim to a directory keeps# the shortest name; ties break on the key so the order never depends on@@ -392,6 +400,12 @@ def _assign_targets(groups: list[ConceptGroup]) -> None:candidates.extend(d for d in group.dirs if d and d not in candidates)if owned and owned not in candidates:candidates.append(owned)+ # The root is substituted before the uniqueness check, never after.+ # Two groups can both reduce to it — one holding the root-level files,+ # another spanning siblings whose only common ancestor is the root —+ # and renaming afterwards would let the second reuse a name the first+ # already holds.+ candidates = [root_name if c == "" else c for c in candidates]# ``None`` rather than "" for "nothing chosen yet": the repository root# is a real, legitimate directory whose path *is* the empty string, and# a falsy check cannot tell it apart from failure. It could not, once —@@ -407,7 +421,8 @@ def _assign_targets(groups: list[ConceptGroup]) -> None:# Every real directory is spoken for. Fall back to the members'# common prefix qualified by the group's identity, which is stable# and unique even though it is no longer only a directory.- chosen = f"{owned}#{group.structural_key.rsplit('-', 1)[-1]}"+ prefix = owned or root_name+ chosen = f"{prefix}#{group.structural_key.rsplit('-', 1)[-1]}"group.target_path = chosentaken.add(chosen)diff --git a/packages/core/src/repowise/core/generation/concept_tree/naming.py b/packages/core/src/repowise/core/generation/concept_tree/naming.pyindex f5a4c8771..731303dfb 100644--- a/packages/core/src/repowise/core/generation/concept_tree/naming.py+++ b/packages/core/src/repowise/core/generation/concept_tree/naming.py@@ -54,9 +54,7 @@ def parse_json_object(content: str) -> dict[str, Any]:"""text = (content or "").strip()if text.startswith("```"):- text = "\n".join(- line for line in text.split("\n") if not line.strip().startswith("```")- )+ text = "\n".join(line for line in text.split("\n") if not line.strip().startswith("```"))try:parsed = json.loads(text)except (json.JSONDecodeError, ValueError):@@ -85,7 +83,17 @@ def _humanise(segment: str) -> str:continue# Split camelCase so ``pageGenerator`` reads as two words.parts = re.findall(r"[A-Z]+(?![a-z])|[A-Z][a-z]*|[a-z]+|\d+", word) or [word]- out.extend(p.capitalize() if p.islower() else p for p in parts)+ cased = [p.capitalize() if p.islower() else p for p in parts]+ # A trailing number belongs to the word it was written against: the+ # ``c4`` directory is the C4 model, and "C 4 Nodes" reads as a+ # tokeniser leaking into a page title.+ merged: list[str] = []+ for part in cased:+ if part.isdigit() and merged:+ merged[-1] += part+ else:+ merged.append(part)+ out.extend(merged)return " ".join(out)@@ -103,7 +111,12 @@ def deterministic_title(group: ConceptGroup, layer_label: str = "") -> str:tail = [_humanise(s) for s in meaningful[-2:]]title = " ".join(tail).strip()if layer_label and len(title.split()) < MIN_TITLE_WORDS:- title = f"{layer_label} {title}".strip()+ # Drop what the label already says. A ``docs`` directory in a "Docs &+ # Tooling" layer was titled "Docs Tooling Docs", which reads as a+ # generation bug rather than a name.+ label_words = {w.lower() for w in layer_label.split()}+ kept = [w for w in title.split() if w.lower() not in label_words]+ title = f"{layer_label} {' '.join(kept)}".strip()if not title:# The repository root has no path segments to name it by. It is a real# place and a common one — top-level entry points live there — so it@@ -117,6 +130,42 @@ def deterministic_title(group: ConceptGroup, layer_label: str = "") -> str:return title+def disambiguate_titles(titled: list[tuple[str, str]]) -> list[str]:+ """Make a set of ``(title, target_path)`` pairs' titles unique.++ Path-derived names collide easily: two packages that each hold a ``ui``+ directory produce the same last-two-segments name. The pages themselves+ stay distinct because identity is structural, but a reader sees two+ identical rows in the tree and cannot tell which is which, and anything+ that keys on a title (interlinking, an overview listing) has to guess.++ A colliding title takes on the next path segment above the ones it+ already used, and falls back to its full target path if even that+ repeats. Input order does not matter: the pairs are resolved in path+ order, so the same set always produces the same names.+ """+ order = sorted(range(len(titled)), key=lambda i: titled[i][1])+ out = list(titled)+ used: set[str] = set()+ for i in order:+ title, target = titled[i]+ if title not in used:+ used.add(title)+ out[i] = (title, target)+ continue+ segments = [s for s in target.split("/") if s]+ candidate = title+ for extra in reversed(segments[:-2] or segments[:-1]):+ candidate = f"{_humanise(extra)} {title}"+ if candidate not in used:+ break+ if candidate in used:+ candidate = f"{title} ({target})"+ used.add(candidate)+ out[i] = (candidate, target)+ return [t for t, _ in out]++def deterministic_scope(group: ConceptGroup) -> str:"""A scope line stating coverage and its boundary, with no model."""# The root directory is the empty string, which is a real directory and@@ -210,9 +259,7 @@ def _sample_names(members: list[str], limit: int) -> list[str]:"""by_dir: dict[str, list[str]] = {}for member in sorted(members):- by_dir.setdefault(member.rsplit("/", 1)[0], []).append(- member.rsplit("/", 1)[-1]- )+ by_dir.setdefault(member.rsplit("/", 1)[0], []).append(member.rsplit("/", 1)[-1])out: list[str] = []order = sorted(by_dir)depth = 0@@ -406,8 +453,9 @@ def decode_response())- named.sort(key=lambda n: (section_rank.get(n.section, len(sections)), n.order,- n.group.target_path))+ named.sort(+ key=lambda n: (section_rank.get(n.section, len(sections)), n.order, n.group.target_path)+ )return named, sorted(set(invented))diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.pyindex f69a12959..7fe968368 100644--- a/packages/core/src/repowise/core/generation/context/assembler.py+++ b/packages/core/src/repowise/core/generation/context/assembler.py@@ -2,6 +2,7 @@from __future__ import annotations+from pathlib import PurePosixPathfrom typing import Anyimport structlog@@ -278,7 +279,7 @@ class ContextAssembler:def assemble_module_page(self,- module_path: str,+ title: str,language: str,file_contexts: list[FilePageContext],graph: Any, # nx.DiGraph@@ -369,8 +370,19 @@ class ContextAssembler:key_classes.append(h)key_classes = key_classes[:10] # cap total+ # Where the page's files actually live: the directories that hold one+ # directly, not every ancestor of every file, so the list reads as the+ # set of places to go and look. Files at the repository root have+ # ``"."`` for a parent and contribute no entry; a page that is only+ # root files therefore has none, and the template says so rather than+ # printing a dot.+ directories = sorted(+ {parent for f in files if (parent := str(PurePosixPath(f).parent)) != "."}+ )+return ModulePageContext(- module_path=module_path,+ title=title,+ directories=directories,language=language,total_symbols=total_symbols,public_symbols=public_symbols,diff --git a/packages/core/src/repowise/core/generation/context/contexts.py b/packages/core/src/repowise/core/generation/context/contexts.pyindex d68433a4c..edfb9d724 100644--- a/packages/core/src/repowise/core/generation/context/contexts.py+++ b/packages/core/src/repowise/core/generation/context/contexts.py@@ -81,7 +81,12 @@ class SymbolSpotlightContext:@dataclassclass ModulePageContext:- module_path: str+ # The page's display title. A concept group spans several directories and+ # is named for what it does, so this is prose ("Ingestion Pipeline") and+ # not a path. Same split as LayerPageContext's layer_name/layer_id: what+ # the reader sees is separate from what the page is keyed by, and only the+ # key is stable across regenerations.+ title: strlanguage: strtotal_symbols: intpublic_symbols: int@@ -90,6 +95,11 @@ class ModulePageContext:dependents: list[str]pagerank_mean: floatfiles: list[str]+ # The directories the page covers, shallowest first. The title says what+ # the page is about; this says where it lives, which is what a reader+ # needs to go and look. Derived from the members rather than passed in, so+ # it cannot disagree with them.+ directories: list[str] = field(default_factory=list)# Graph intelligence enrichmentfile_summaries: dict[str, str] = field(default_factory=dict)community_label: str = ""diff --git a/packages/core/src/repowise/core/generation/models.py b/packages/core/src/repowise/core/generation/models.pyindex c957a3982..4850a283d 100644--- a/packages/core/src/repowise/core/generation/models.py+++ b/packages/core/src/repowise/core/generation/models.py@@ -98,7 +98,6 @@ class GenerationConfig:coverage_pct: float = 0.20file_page_share: float = 0.50symbol_spotlight_share: float = 0.15- module_page_share: float = 0.10api_contract_share: float = 0.08infra_page_share: float = 0.05scc_share: float = 0.04@@ -113,17 +112,6 @@ class GenerationConfig:file_page_min_symbols: int = 1skip_trivial_files: bool = Truededupe_near_clones: bool = True- # Module-page grouping source. "curated" (default) groups by the wiki- # modules the KG curation pass derives (stable path ids, human names,- # right-sized groups) and silently falls back to "community" when no- # curated modules are available (curation off, degraded, or no KG- # artifact), so it is always safe. "community" groups by raw graph- # communities (the pre-curation behavior, kept as the escape hatch),- # "top_dir" by top-level directory.- # min_module_size is the floor below which a group doesn't get its own- # page (its files still appear under file_page).- module_grouping: Literal["community", "top_dir", "curated"] = "curated"- min_module_size: int = 3# Phase 3: emit the curated Onboarding collection at level 8. Each# subkind defines its own gate; slots whose gates fail are silently# skipped (no UI nav entry either).diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.pyindex d202d5001..c193e3662 100644--- a/packages/core/src/repowise/core/generation/page_generator/levels.py+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py@@ -278,6 +278,15 @@ def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:for mg in run.sel_module_groups:fcs = [run.file_page_contexts[fp] for fp in mg.file_paths if fp in run.file_page_contexts]if not fcs:+ # A concept group whose files all failed to build a context. The+ # partition is total, so this is a hole in the tree rather than a+ # page not worth writing, and it should be visible when it+ # happens instead of leaving the reader to notice the gap.+ log.warning(+ "module_page.skipped_no_file_contexts",+ target_path=mg.key,+ members=len(mg.file_paths),+ )continuepage_id = compute_page_id("module_page", mg.key)if not run._emit(page_id):@@ -300,6 +309,8 @@ def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:community_label=mg.label,community_cohesion=mg.cohesion,target_path=mg.key,+ structural_key=mg.structural_key,+ members=list(mg.file_paths),),))diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.pyindex dfff264c3..78657045d 100644--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py@@ -256,8 +256,9 @@ class _GenerationRun:kg_file_scores=kg_scores or None,# Curated wiki modules: prefer the in-memory pipeline# result (fresh); the artifact file is absent on first init- # and one run stale on update. Inert unless- # module_grouping == "curated".+ # and one run stale on update. Read only for the file -> layer+ # map that steers concept grouping, so absence costs taste and+ # never coverage.kg_modules=self.kg_modules or self.kg_ctx.get_modules() or None,# Per-file question demand from session transcripts: tilts the# file_page budget toward modules agents ask about. Empty (nodiff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.pyindex 0c76e6091..6d2b1b1a1 100644--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py@@ -96,7 +96,7 @@ class PerTypeGenerationMixin:async def generate_module_page(self,- module_path: str,+ title: str,language: str,file_contexts: list[FilePageContext],graph: Any,@@ -108,9 +108,11 @@ class PerTypeGenerationMixin:community_label: str | None = None,community_cohesion: float | None = None,target_path: str | None = None,+ structural_key: str = "",+ members: list[str] | None = None,) -> GeneratedPage:ctx = self._assembler.assemble_module_page(- module_path,+ title,language,file_contexts,graph,@@ -140,17 +142,32 @@ class PerTypeGenerationMixin:"most_active_file": most_active.get("file_path", ""),"most_active_commits_90d": most_active.get("commit_count_90d", 0),}- page_target = target_path or module_path+ # The page id is built from this, so it has to be a path, and it has+ # to be the same path the grouper chose: it computes one shared+ # directory per group and guarantees the result is unique across+ # groups. Picking a different one here (the first directory, say)+ # would mint an id the rest of the run does not agree with, so a+ # caller that supplies no target gets the group's own anchor or, for+ # a group with no directory at all, the root name the grouper uses.+ page_target = target_path or (min(ctx.directories) if ctx.directories else "root")# The files this page covers. Its own identity and its place in the- # tree both derive from them: a module's target_path can be a- # clustering ordinal rather than a directory, in which case the member- # list is the only thing that says where the page belongs.- members = sorted(fc.file_path for fc in file_contexts)+ # tree both derive from them: the page groups several directories, so+ # its target_path is one directory the group touches rather than a+ # container for all of it, and the member list is the only thing that+ # says where the page belongs.+ #+ # Taken from the group rather than from ``file_contexts``, and the+ # difference is load-bearing. A file context only exists for a file the+ # run built one for, so deriving members here would silently narrow the+ # page to that subset while ``structural_key`` still hashed the whole+ # group. Placement resolves file ownership from this list, so the+ # dropped files would be parented somewhere else entirely, and the two+ # records of what the page covers would disagree.+ covered = sorted(members) if members else sorted(fc.file_path for fc in file_contexts)if self._config.deterministic:- page = self._deterministic_module_page(- ctx, page_target, f"Module: {module_path}", module_git_summary- )- page.metadata["file_paths"] = members+ page = self._deterministic_module_page(ctx, page_target, title, module_git_summary)+ page.metadata["file_paths"] = covered+ page.structural_key = structural_key or page.structural_keyreturn pageuser_prompt = self._render("module_page.j2", ctx=ctx, module_git_summary=module_git_summary)response = await self._call_provider(@@ -159,12 +176,17 @@ class PerTypeGenerationMixin:page = self._build_generated_page("module_page",page_target,- f"Module: {module_path}",+ title,response,compute_source_hash(user_prompt),GENERATION_LEVELS["module_page"],)- page.metadata["file_paths"] = members+ page.metadata["file_paths"] = covered+ # Set by the producer, preserved by ``_stamp_structural_keys``. The+ # grouper hashed exactly this member list when it decided what the+ # group was, so recomputing it downstream would give two places that+ # must agree about page identity — the arrangement D2 rules out.+ page.structural_key = structural_key or page.structural_keyreturn pageasync def generate_scc_page(diff --git a/packages/core/src/repowise/core/generation/related_pages.py b/packages/core/src/repowise/core/generation/related_pages.pyindex 209544637..697c0e365 100644--- a/packages/core/src/repowise/core/generation/related_pages.py+++ b/packages/core/src/repowise/core/generation/related_pages.py@@ -61,7 +61,8 @@ def _module_siblings(for group in module_groups or []:paths = list(getattr(group, "file_paths", ()) or ())for path in paths:- # First group wins — curated groups precede fallbacks upstream.+ # First group wins. The concept partition is disjoint, so this+ # only matters if a caller ever passes overlapping groups.siblings.setdefault(path, [p for p in paths if p != path])return siblingsdiff --git a/packages/core/src/repowise/core/generation/selection/__init__.py b/packages/core/src/repowise/core/generation/selection/__init__.pyindex cc3abe266..ee1f44ce2 100644--- a/packages/core/src/repowise/core/generation/selection/__init__.py+++ b/packages/core/src/repowise/core/generation/selection/__init__.py@@ -23,7 +23,6 @@ from .scoring import (score_api_contract,score_file,score_infra,- score_module,score_scc,score_symbol,)@@ -46,7 +45,6 @@ __all__ = ["score_api_contract","score_file","score_infra",- "score_module","score_scc","score_symbol","select_pages",diff --git a/packages/core/src/repowise/core/generation/selection/scoring.py b/packages/core/src/repowise/core/generation/selection/scoring.pyindex 9fbc7fd05..91ca58910 100644--- a/packages/core/src/repowise/core/generation/selection/scoring.py+++ b/packages/core/src/repowise/core/generation/selection/scoring.py@@ -15,7 +15,6 @@ from __future__ import annotationsfrom pathlib import Pathfrom typing import Any-# -- File score --------------------------------------------------------# Bonuses applied additively on top of normalized PageRank+betweenness.@@ -123,22 +122,6 @@ def score_symbol(symbol: Any, file_pagerank: float, max_pagerank: float) -> floareturn _normalize(file_pagerank, max_pagerank) * weight-# -- Module score ---------------------------------------------------------def score_module(*, size: int, cohesion: float, min_module_size: int) -> float:- """Score a community/directory cluster for ``module_page`` candidacy.-- Communities below ``min_module_size`` score 0 — they fold into the- parent directory's module page (or are dropped if there is no- parent).- """- if size < min_module_size:- return 0.0- # Cohesion is typically [0, 1]. Multiply by log-ish size factor.- return (size**0.5) * (0.5 + cohesion)--# -- SCC score ---------------------------------------------------------diff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.pyindex 80369db07..a854fd56f 100644--- a/packages/core/src/repowise/core/generation/selection/selector.py+++ b/packages/core/src/repowise/core/generation/selection/selector.py@@ -8,7 +8,8 @@ emitted.from __future__ import annotations-from dataclasses import dataclass, field+from collections import Counter+from dataclasses import dataclass, field, replacefrom pathlib import Pathfrom typing import Any@@ -16,6 +17,8 @@ import structlogfrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY+from ..concept_tree.grouping import group_files+from ..concept_tree.naming import deterministic_title, disambiguate_titlesfrom ..models import scc_page_slugfrom ..tour import DEFAULT_MAX_LANDMARKS, tour_landmark_pathsfrom .budget import (@@ -29,7 +32,6 @@ from .scoring import (score_api_contract,score_file,score_infra,- score_module,score_scc,score_symbol,)@@ -50,10 +52,17 @@ _CODE_LANGUAGES = _LANG_REGISTRY.code_languages()class ModuleGroup:"""One ``module_page`` worth of files.- ``key`` is the stable identifier persisted as ``target_path``:- the module's real directory path when grouping by curated KG modules- (so path-prefix child lookups work), ``community-<id>`` when grouping- by community, the top-level directory when falling back to ``top_dir``.+ ``key`` is the stable identifier persisted as ``target_path``, and it is+ always a real directory: the shallowest directory the group's members+ share. The bench gold set matches pages by exact equality against the+ surfaced ``target_path``, so a group that merged several sibling+ directories still has to name one of them.++ ``structural_key`` is where the abstract identity lives — a hash of the+ sorted member list, carrying the ``concept-`` prefix minted by the+ grouper. It is set by whatever produced the group rather than recomputed+ downstream, because two places agreeing about page identity is the+ arrangement D2 exists to prevent."""key: str@@ -62,6 +71,7 @@ class ModuleGroup:file_paths: tuple[str, ...]label: str | None = Nonecohesion: float | None = None+ structural_key: str = ""@dataclass@@ -127,9 +137,10 @@ class SelectionInputs:config: Any # GenerationConfig — duck-typed to avoid the import cyclekg_file_scores: dict[str, float] | None = None# Curated wiki modules from the KG artifact (``modules`` top-level key),- # passed through — never re-derived here. Only read when- # ``config.module_grouping == "curated"``; ``None``/empty falls back to- # community grouping (the fallback-matrix "degraded" row).+ # passed through — never re-derived here. Read for one thing only: the+ # file -> layer map that steers which adjacent directories merge into a+ # concept group. Absent or partial degrades the grouping's taste, never+ # its coverage, because the partition itself needs no KG input.kg_modules: list[dict] | None = None# Per-file question demand mined from session transcripts# (``core.sessions.miners.demand.aggregate_file_demand``): repo-relative@@ -182,24 +193,6 @@ def _passes_tail_floor(path: str, tail_dirs: tuple[str, ...] | None) -> bool:return True-def _is_test_group(files: list[Any]) -> bool:- """Whether a module group is mostly test files.-- Test directories do not deserve a module page. The same floor already- applies to the deterministic coverage tail, where excluding test files- raised retrieval rather than lowering it: they add pages that match on- vocabulary without carrying an answer. Tests keep their own file pages,- which is what an agent actually needs when it goes looking for one.-- Majority rather than any, so a production module with a colocated test- beside it (the Go and Jest convention) keeps its page.- """- if not files:- return False- tests = sum(1 for p in files if getattr(p.file_info, "is_test", False))- return tests * 2 > len(files)--def _select_deterministic_tail(files: list[tuple[float, str]],selected_files: list[str],@@ -287,136 +280,94 @@ def _build_symbol_candidates(return deduped-def _build_curated_module_groups(- inputs: SelectionInputs, min_size: int-) -> list[tuple[float, ModuleGroup]] | None:- """Scored groups from curated KG modules, or ``None`` when unavailable.+def _layer_map_from_kg(+ inputs: SelectionInputs,+) -> tuple[dict[str, str], dict[str, str]]:+ """File -> curated layer id, and layer id -> display name.- ``key`` = the module's real directory path (its ``target_path``, so the- MCP child lookup ``target_path LIKE 'dir/%'`` works), ``display``/``label``- = the curated human name, ``cohesion`` = None (the community block in the- template is conditional and has no meaning for directory groups). Ranked- by Σ PageRank of members — better than ``score_module``'s cohesion term,- which is meaningless here. Returns ``None`` only when no curated modules- were passed in (→ community fallback); an artifact whose modules all fall- below the floor yields an empty list, not a vocabulary mix.+ The layer signal only steers which adjacent runs of directories merge; it+ never forces a split, so an absent or partial map degrades the grouping's+ taste rather than its correctness. That is why this reads whatever the KG+ artifact happens to carry instead of requiring it."""- if not inputs.kg_modules:- return None-- code_by_path = {p.file_info.path: p for p in inputs.parsed_files if _is_code_file(p)}- scored: list[tuple[float, ModuleGroup]] = []- seen_keys: set[str] = set()- for module in inputs.kg_modules:- if module.get("wholeLayer"):- # 1:1 with a layer page (single-module layers, flat libs) — a- # module doc would re-document the layer. Skip the page; the- # module stays in the KG artifact for canvas/coverage.+ layer_of_file: dict[str, str] = {}+ for module in inputs.kg_modules or []:+ layer_id = module.get("layerId")+ if not isinstance(layer_id, str) or not layer_id:continue- member_paths = sorted(- path- for nid in module.get("nodeIds", [])- if isinstance(nid, str) and (path := nid.removeprefix("file:")) in code_by_path- )- if len(member_paths) < min_size:- continue- key = module.get("path") or (module.get("id") or "").removeprefix("module:")- if not key or key in seen_keys:- # Rare cross-layer dir collision: the slug id is still unique.- key = (module.get("id") or "").removeprefix("module:")- if not key or key in seen_keys:- continue- seen_keys.add(key)- name = module.get("name") or key- language = module.get("language") or code_by_path[member_paths[0]].file_info.language- score = sum(inputs.pagerank.get(p, 0.0) for p in member_paths)- scored.append(- (- score,- ModuleGroup(- key=key,- display=name,- language=language,- file_paths=tuple(member_paths),- label=name,- cohesion=None,- ),- )- )- scored.sort(key=lambda x: (-x[0], x[1].key))- return scored+ for nid in module.get("nodeIds", []):+ if isinstance(nid, str) and nid.startswith("file:"):+ layer_of_file[nid[5:]] = layer_id+ labels = {+ lid: lid.removeprefix("layer:").replace("-", " ").replace("_", " ").title()+ for lid in set(layer_of_file.values())+ }+ return layer_of_file, labelsdef _build_module_groups(inputs: SelectionInputs) -> list[tuple[float, ModuleGroup]]:- """Return scored module groups — curated, community-based, or top-dir."""- cfg = inputs.config- min_size = max(1, getattr(cfg, "min_module_size", 3))- grouping = getattr(cfg, "module_grouping", "community")-- if grouping == "curated":- curated = _build_curated_module_groups(inputs, min_size)- if curated is not None:- return curated- grouping = "community" # no curated artifact → today's path, unchanged-- use_communities = grouping == "community"-- # Bucket files into groups.- groups: dict[str, list[Any]] = {}- group_lang: dict[str, str] = {}- group_label: dict[str, str | None] = {}- group_cohesion: dict[str, float | None] = {}- group_display: dict[str, str] = {}-- if use_communities and inputs.community_info:- for p in inputs.parsed_files:- if not _is_code_file(p):- continue- cid = inputs.community.get(p.file_info.path)- if cid is None:- continue- key = f"community-{cid}"- groups.setdefault(key, []).append(p)- group_lang.setdefault(key, p.file_info.language)- if key not in group_display:- ci = inputs.community_info.get(cid)- label = getattr(ci, "label", "") or f"cluster_{cid}"- group_display[key] = label- group_label[key] = label- group_cohesion[key] = float(getattr(ci, "cohesion", 0.0) or 0.0)- else:- for p in inputs.parsed_files:- if not _is_code_file(p):- continue- parts = Path(p.file_info.path).parts- key = parts[0] if len(parts) > 1 else "root"- groups.setdefault(key, []).append(p)- group_lang.setdefault(key, p.file_info.language)- group_display.setdefault(key, key)- group_label.setdefault(key, None)- group_cohesion.setdefault(key, None)+ """Return the concept groups, scored, one per ``module_page``.++ This is the concept tree, not one page per directory. The partition comes+ from :func:`group_files`, which bins the production files into bounded,+ path-local subtrees with no model in the loop, so it is total (every+ non-test production file lands in exactly one group) and deterministic+ (the same file set always produces the same groups, and therefore the same+ page identities).++ Totality is the property that makes this a replacement rather than an+ alternative. The per-directory grouping this supersedes covered a subset+ of the tree under a worse grouping and gave eleven of its pages to test+ directories, which D8 measured as pure dilution. There is no coexistence+ mode: D1 forbids one page type existing in two forms, and a wiki holding+ both populations would be exactly that.++ Scored by summed PageRank so the most central subsystem sorts first. The+ score no longer rations anything — see :func:`select_pages` — but page+ order is still what the reader sees first.+ """+ files = [+ p.file_info.path+ for p in inputs.parsed_files+ if _is_code_file(p) and not getattr(p.file_info, "is_test", False)+ ]+ if not files:+ return []+ layer_of_file, layer_labels = _layer_map_from_kg(inputs)+ groups = group_files(files, layer_of_file=layer_of_file)++ lang_of = {p.file_info.path: p.file_info.language for p in inputs.parsed_files}+ # Names are decided over the whole set, not per group: two packages that+ # each hold a ``ui`` directory derive the same name from their paths, and+ # two identical rows in the tree are indistinguishable to a reader even+ # though the pages behind them are not.+ titles = disambiguate_titles(+ [+ (deterministic_title(g, layer_labels.get(g.dominant_layer, "")), g.target_path)+ for g in groups+ ]+ )scored: list[tuple[float, ModuleGroup]] = []- for key, files in groups.items():- if _is_test_group(files):- continue- s = score_module(- size=len(files),- cohesion=group_cohesion.get(key) or 0.0,- min_module_size=min_size,- )- if s <= 0.0:- continue+ for group, title in zip(groups, titles, strict=True):+ langs = Counter(lang_of.get(m, "") for m in group.members)+ langs.pop("", None)+ score = sum(inputs.pagerank.get(m, 0.0) for m in group.members)scored.append((- s,+ score,ModuleGroup(- key=key,- display=group_display.get(key, key),- language=group_lang.get(key, "unknown"),- file_paths=tuple(sorted(p.file_info.path for p in files)),- label=group_label.get(key),- cohesion=group_cohesion.get(key),+ # Taken as given. The grouper guarantees it is non-empty+ # and distinct across groups, and naming the root here+ # instead would sit outside the uniqueness check that+ # makes the guarantee.+ key=group.target_path,+ display=title,+ language=langs.most_common(1)[0][0] if langs else "unknown",+ file_paths=tuple(group.members),+ label=None,+ cohesion=None,+ structural_key=group.structural_key,),))@@ -469,7 +420,10 @@ def _shares_from_config(cfg: Any) -> dict[str, float]:return {"file_page": getattr(cfg, "file_page_share", 0.50),"symbol_spotlight": getattr(cfg, "symbol_spotlight_share", 0.15),- "module_page": getattr(cfg, "module_page_share", 0.10),+ # Zero, and not a knob. The concept pages are taken whole, so a share+ # here would reserve budget for a bucket that does not spend it and+ # quietly shrink the file-page coverage the user actually asked for.+ "module_page": 0.0,"api_contract": getattr(cfg, "api_contract_share", 0.08),"infra_page": getattr(cfg, "infra_page_share", 0.05),"scc_page": getattr(cfg, "scc_share", 0.04),@@ -691,6 +645,11 @@ def select_pages(inputs: SelectionInputs) -> Selection:shares=_shares_from_config(cfg),n_files=len(inputs.parsed_files),)+ # The module bucket is not rationed (see the Selection below), so the+ # allocation has to say so. Leaving the budgeted number here would make+ # the cost estimator quote a figure the run does not honour, which is the+ # one thing the "allow-set honored verbatim" contract exists to prevent.+ allocation = replace(allocation, module_page=len(modules))# File pages, tilted toward the modules agents ask about most (demand-free# top-K when there is no session data, so fresh installs are unchanged).@@ -720,7 +679,14 @@ def select_pages(inputs: SelectionInputs) -> Selection:file_page_paths=selected_files,deterministic_tail_paths=deterministic_tail,symbol_spotlights=[t for _, t in symbols[: allocation.symbol_spotlight]],- module_groups=[m for _, m in modules[: allocation.module_page]],+ # Never truncated, unlike every other bucket. The concept groups are a+ # total partition of the production files, so rationing them does not+ # produce a smaller wiki — it produces one with files that belong to no+ # page at all, and the tree stops being a map of the repository. The+ # cost of not rationing is bounded by construction rather than by a+ # budget: the grouper's size ladder holds the page count near 50-80 for+ # a repository this size and scales sublinearly above it.+ module_groups=[m for _, m in modules],api_contract_paths=[p for _, p in apis[: allocation.api_contract]],infra_paths=[p for _, p in infras[: allocation.infra_page]],scc_groups=[g for _, g in sccs[: allocation.scc_page]],diff --git a/packages/core/src/repowise/core/generation/templates/deterministic/module_page.j2 b/packages/core/src/repowise/core/generation/templates/deterministic/module_page.j2index 09ae38a7e..e8312322d 100644--- a/packages/core/src/repowise/core/generation/templates/deterministic/module_page.j2+++ b/packages/core/src/repowise/core/generation/templates/deterministic/module_page.j2@@ -5,7 +5,9 @@Overview is synthesised from the module's own signals (centrality, entrypoints, cluster) and the structural sections carry the rest.-#}-# Module: {{ ctx.module_path }}+# {{ ctx.title }}++{% if ctx.directories %}{% for d in ctx.directories[:6] %}`{{ d }}`{% if not loop.last %} · {% endif %}{% endfor %}{% if ctx.directories | length > 6 %} · and {{ ctx.directories | length - 6 }} more{% endif %}{% else %}Repository root{% endif %}**Language:** {{ ctx.language }} | **Files:** {{ ctx.files | length }} | **Public symbols:** {{ ctx.public_symbols }} / {{ ctx.total_symbols }}{% if ctx.community_label %} | **Cluster:** {{ ctx.community_label }}{% if ctx.community_cohesion > 0 %} (cohesion {{ "%.2f"|format(ctx.community_cohesion) }}){% endif %}{% endif %}@@ -16,7 +18,7 @@{%- if ctx.pagerank_mean is defined and ctx.pagerank_mean >= 0.005 %}{% set _ = _role.append("sits centrally in the import graph (mean PageRank " ~ "%.4f"|format(ctx.pagerank_mean) ~ ")") %}{% endif -%}{%- if ctx.dependents %}{% set _ = _role.append("is imported by " ~ ctx.dependents | length ~ " other module" ~ ("s" if ctx.dependents | length != 1 else "")) %}{% endif -%}{%- if ctx.dependencies %}{% set _ = _role.append("draws on " ~ ctx.dependencies | length ~ " internal module" ~ ("s" if ctx.dependencies | length != 1 else "")) %}{% endif -%}-`{{ ctx.module_path }}` groups {{ ctx.files | length }} {{ ctx.language }} file{{ "s" if ctx.files | length != 1 else "" }} exposing {{ ctx.public_symbols }} public symbol{{ "s" if ctx.public_symbols != 1 else "" }}.{% if _role %} It {{ _role | join("; ") }}.{% endif %}+{{ ctx.title }} covers {{ ctx.files | length }} {{ ctx.language }} file{{ "s" if ctx.files | length != 1 else "" }}{% if ctx.directories %} across {{ ctx.directories | length }} director{{ "ies" if ctx.directories | length != 1 else "y" }}{% endif %}, exposing {{ ctx.public_symbols }} public symbol{{ "s" if ctx.public_symbols != 1 else "" }}.{% if _role %} It {{ _role | join("; ") }}.{% endif %}{% if ctx.entry_points %}## Entry Pointsdiff --git a/packages/core/src/repowise/core/generation/templates/module_page.j2 b/packages/core/src/repowise/core/generation/templates/module_page.j2index b05f554fd..ecbcbe5da 100644--- a/packages/core/src/repowise/core/generation/templates/module_page.j2+++ b/packages/core/src/repowise/core/generation/templates/module_page.j2@@ -1,8 +1,8 @@-You are documenting the module **{{ ctx.module_path }}** (language: {{ ctx.language }}).+You are documenting **{{ ctx.title }}** (language: {{ ctx.language }}), a group of files that belong together, not a single directory. Write about what the group does as a whole.## Module Overview-**Module:** `{{ ctx.module_path }}` | **Language:** {{ ctx.language }} | **Files:** {{ ctx.files | length }} | **Public symbols:** {{ ctx.public_symbols }} / {{ ctx.total_symbols }}{% if ctx.community_label %} | **Community:** {{ ctx.community_label }}{% if ctx.community_cohesion > 0 %} (cohesion {{ "%.2f"|format(ctx.community_cohesion) }}){% endif %}{% endif %}+**Covers:** {% if ctx.directories %}{% for d in ctx.directories[:6] %}`{{ d }}`{% if not loop.last %}, {% endif %}{% endfor %}{% if ctx.directories | length > 6 %} and {{ ctx.directories | length - 6 }} more{% endif %}{% else %}the repository root{% endif %} | **Language:** {{ ctx.language }} | **Files:** {{ ctx.files | length }} | **Public symbols:** {{ ctx.public_symbols }} / {{ ctx.total_symbols }}{% if ctx.community_label %} | **Community:** {{ ctx.community_label }}{% if ctx.community_cohesion > 0 %} (cohesion {{ "%.2f"|format(ctx.community_cohesion) }}){% endif %}{% endif %}{# The role-leading sentence is the highest-leverage signal a module pagecarries. Surface the module's position in the system so the LLM can framediff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.pyindex 725b0d3dd..baa532d55 100644--- a/packages/core/src/repowise/core/ingestion/traverser.py+++ b/packages/core/src/repowise/core/ingestion/traverser.py@@ -21,7 +21,7 @@ import threadingfrom collections.abc import Iteratorfrom dataclasses import dataclass, fieldfrom datetime import datetime-from pathlib import Path+from pathlib import Path, PurePosixPathimport pathspecimport structlog@@ -197,6 +197,28 @@ _ENTRY_POINT_NAME_SUFFIXES: tuple[str, ...] = tuple(sorted(p[1:] for p in _LANG_REGISTRY.entry_point_names() if p.startswith("*")))+# Test-file conventions, from the same registry each language declares them+# on. ``spec_`` is carried over from the rule these replaced: the registry+# has the ``_spec`` suffix and the ``spec_helper`` stem but no prefix, and+# dropping it would unclassify RSpec-style ``spec_foo.rb``.+_TEST_STEM_PREFIXES: tuple[str, ...] = tuple(+ sorted({*_LANG_REGISTRY.test_stem_prefixes(), "spec_"})+)+_TEST_STEM_SUFFIXES: tuple[str, ...] = _LANG_REGISTRY.test_stem_suffixes()+_TEST_INFIXES: tuple[str, ...] = _LANG_REGISTRY.test_infixes()+_TEST_FIXTURE_STEMS: frozenset[str] = _LANG_REGISTRY.test_fixture_stems()+# Directory names that mark everything beneath them as test material.+# ``__tests__`` is the JavaScript convention and is the reason this set is+# not simply the registry's per-language tokens.+_TEST_DIR_SEGMENTS: frozenset[str] = frozenset(+ {"test", "tests", "__tests__", "__test__"}+ | {+ token+ for tokens in _LANG_REGISTRY.test_dir_tokens_by_language().values()+ for token in tokens+ }+)+# Default file-size limit_DEFAULT_MAX_FILE_SIZE_BYTES: int = 500 * 1024 # 500 KB@@ -650,13 +672,29 @@ def _is_generated(abs_path: Path) -> bool:def _is_test_file(rel_path: str, filename: str) -> bool:- stem = Path(filename).stem.lower()- if stem.startswith("test_") or stem.endswith("_test"):- return True- if stem.startswith("spec_") or stem.endswith("_spec"):+ """Whether *rel_path* is test material, by directory or by filename.++ Conventions come from the language registry, which is where each+ language already declares them, rather than from a second hard-coded+ list. The two rules this replaced knew only ``test_x`` / ``x_test`` and a+ slash-delimited ``/tests/`` substring, so both halves of the JavaScript+ convention (``__tests__/`` directories and ``x.test.ts`` filenames) and+ every root-level suite (``tests/conftest.py``) read as production code to+ everything downstream of this flag.+ """+ name = filename.lower()+ stem = PurePosixPath(name).stem+ if (+ stem in _TEST_FIXTURE_STEMS+ or stem.startswith(_TEST_STEM_PREFIXES)+ or stem.endswith(_TEST_STEM_SUFFIXES)+ or any(infix in name for infix in _TEST_INFIXES)+ ):return True- path_lower = rel_path.lower()- return "/test/" in path_lower or "/tests/" in path_lower or "/spec/" in path_lower+ # Directory segments, including the first: a repository-root ``tests/``+ # is the usual layout, not the exception.+ segments = rel_path.lower().replace("\\", "/").split("/")[:-1]+ return any(seg in _TEST_DIR_SEGMENTS for seg in segments)def _is_config_file(language: LanguageTag) -> bool:diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex 3424b8c18..ff9e84556 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -298,9 +298,7 @@ async def run_pipeline(try:graph_builder, git_meta_map = await resume_controller.rehydrate_index(repo_path)if progress:- progress.on_message(- "info", " ↳ Resuming — reusing persisted graph + git index"- )+ progress.on_message("info", " ↳ Resuming — reusing persisted graph + git index")(parsed_files,file_infos,@@ -601,8 +599,8 @@ async def run_pipeline(# ---- Phase 3: Generation (optional) ------------------------------------generated_pages: list[Any] | None = None# Structural page types this run was authoritative for (see- # PipelineResult.authoritative_page_types). Stays empty unless curated- # generation engaged below.+ # PipelineResult.authoritative_page_types). Stays empty unless generation+ # runs below, because a run that generated nothing decided nothing.authoritative_page_types: set[str] = set()# DETERMINISTIC supplies its own null provider so a caller with no key can# still get a wiki. Scoped to this phase rather than assigned over@@ -642,9 +640,7 @@ async def run_pipeline(if mode.deterministic_docs and not resolved_generation_config.deterministic:from dataclasses import replace as _dc_replace- resolved_generation_config = _dc_replace(- resolved_generation_config, deterministic=True- )+ resolved_generation_config = _dc_replace(resolved_generation_config, deterministic=True)# Phase 2 enrichment: flag framework-defined HTTP surfaces (FastAPI,# ASP.NET controllers, …) as api_contract so they route through the@@ -727,9 +723,7 @@ async def run_pipeline(else None),kg_data=(- knowledge_graph_result.to_dict()- if knowledge_graph_result is not None- else None+ knowledge_graph_result.to_dict() if knowledge_graph_result is not None else None),)except BaseException:@@ -744,19 +738,33 @@ async def run_pipeline(# Record which structural page types this run authoritatively decided,# so the sweep can retire prior rows of a type even when this run- # legitimately emitted zero pages of it. Mirrors the selector's curated- # engagement test (_build_curated_module_groups returns None — i.e.- # falls back to community — only when kg_modules is empty) so the signal- # is set iff the curated grouping actually engaged. On the degraded- # community fallback both stay unset, preserving degradation honesty.- kg_modules_present = bool(- knowledge_graph_result is not None and knowledge_graph_result.modules+ # legitimately emitted zero pages of it.+ #+ # ``module_page`` needs no KG and no budget: the concept grouping+ # partitions every production file, so a run that produced any concept+ # page produced all of them and prior rows should go. The condition+ # that used to stand here tested whether a curated KG artifact had+ # engaged a grouping mode that no longer exists, which stranded the+ # superseded pages on exactly the repositories the KG failed on.+ #+ # It is still not unconditional, because "emitted zero" has causes+ # other than "there is nothing to document": ``file_pages_only``+ # returns before the concept level runs at all, and a provider error+ # on that level is logged per page rather than raised. Claiming+ # authority there would delete every concept page over a 429. So the+ # claim is made on evidence — at least one page produced — with the+ # genuinely empty repository as the one exception, since it has no+ # page to show and its stale rows are exactly what wants sweeping.+ produced_module_page = any(+ getattr(page, "page_type", "") == "module_page" for page in (generated_pages or [])+ )+ has_code_to_group = any(+ not getattr(p.file_info, "is_test", False) for p in (parsed_files or []))kg_layers_present = bool(knowledge_graph_result is not None and knowledge_graph_result.layers)- module_grouping = getattr(resolved_generation_config, "module_grouping", "community")- if module_grouping == "curated" and kg_modules_present:+ if produced_module_page or not has_code_to_group:authoritative_page_types.add("module_page")if kg_layers_present:authoritative_page_types.add("layer_page")diff --git a/tests/unit/cli/test_workspace_curated_grouping.py b/tests/unit/cli/test_workspace_kg_artifact_wiring.pysimilarity index 82%rename from tests/unit/cli/test_workspace_curated_grouping.pyrename to tests/unit/cli/test_workspace_kg_artifact_wiring.pyindex 26d1538f6..c46b06978 100644--- a/tests/unit/cli/test_workspace_curated_grouping.py+++ b/tests/unit/cli/test_workspace_kg_artifact_wiring.py@@ -1,10 +1,11 @@-"""Tests for curated-grouping wiring in ``repowise workspace add``.--``module_grouping="curated"`` is the default. Doc generation can only engage-curated module grouping if (a) the KG artifact (``.repowise/knowledge-graph.json``)-was saved during indexing and (b) the generator is told the ``repo_path`` so it-can load that artifact. These tests pin both seams so a regression to community-grouping is caught.+"""Tests for knowledge-graph wiring in ``repowise workspace add``.++Doc generation reads the curated KG for the file -> layer map that steers+which adjacent directories merge into a concept page. It gets there only if+(a) the artifact (``.repowise/knowledge-graph.json``) was saved during+indexing and (b) the generator is told the ``repo_path`` so it can load it.+Neither seam changes coverage if it breaks, which is exactly why a break is+easy to miss: the wiki still has a page for every file, grouped worse."""from __future__ import annotations@@ -16,7 +17,7 @@ from repowise.cli.commands.workspace_cmd import _generate_docs_for_added_repodef test_do_index_save_idiom_writes_artifact(tmp_path):- """The ``_do_index`` save idiom must persist the curated KG artifact.+ """The ``_do_index`` save idiom must persist the KG artifact.``_do_index`` is a nested closure, so we exercise the exact save call itperforms: when the pipeline result carries a ``knowledge_graph_result``,@@ -35,7 +36,7 @@ def test_do_index_save_idiom_writes_artifact(tmp_path):def test_generate_docs_for_added_repo_passes_repo_path(tmp_path):- """``generate_all`` must receive ``repo_path`` so the curated KG loads."""+ """``generate_all`` must receive ``repo_path`` so the KG artifact loads."""repo_path = tmp_pathgenerator = MagicMock()diff --git a/tests/unit/generation/test_concept_grouping.py b/tests/unit/generation/test_concept_grouping.pyindex 1720496e8..6cdfb8e37 100644--- a/tests/unit/generation/test_concept_grouping.py+++ b/tests/unit/generation/test_concept_grouping.py@@ -119,9 +119,7 @@ class TestStability:def test_input_order_does_not_change_the_grouping(self):files = _repo(SPEC)forward = [g.structural_key for g in group_files(files, params=TINY)]- reverse = [- g.structural_key for g in group_files(list(reversed(files)), params=TINY)- ]+ reverse = [g.structural_key for g in group_files(list(reversed(files)), params=TINY)]assert forward == reversedef test_one_added_file_does_not_reshuffle_the_tree(self):@@ -134,8 +132,7 @@ class TestStability:files = _repo(SPEC)before = {g.structural_key for g in group_files(files, params=TINY)}after = {- g.structural_key- for g in group_files([*files, "src/alpha/new_thing.py"], params=TINY)+ g.structural_key for g in group_files([*files, "src/alpha/new_thing.py"], params=TINY)}assert len(before - after) == 1, "more than one group changed identity"assert len(after - before) == 1@@ -195,26 +192,47 @@ class TestIdentity:def test_target_path_stays_a_real_directory(self):files = _repo(SPEC)- directories = {f.rsplit("/", 1)[0] if "/" in f else "" for f in files}+ # The repository root is a real directory whose path is the empty+ # string, and the empty string cannot be a page id, so it is the one+ # target that is a name rather than a path.+ directories = {f.rsplit("/", 1)[0] for f in files if "/" in f} | {"root"}for group in group_files(files, params=TINY):- assert group.target_path in directories, (- f"{group.target_path!r} is not a directory in this repository"- )+ assert (+ group.target_path in directories+ ), f"{group.target_path!r} is not a directory in this repository"- def test_a_group_of_root_level_files_is_targeted_at_the_root(self):+ def test_a_group_of_root_level_files_is_named_for_the_root(self):"""The repository root is a directory whose path is the empty string.Treating that as "no directory found" sent the group to theidentity-hash fallback, which then became its page id, its title- ("D 73789 F 001 Bd") and its scope sentence.+ ("D 73789 F 001 Bd") and its scope sentence. It cannot keep the empty+ string either, because the page id is built from the target and an+ empty one mints ``module_page:``, so it gets a name here where+ uniqueness is enforced."""files = _repo({"": 3, "src/pkg": 10})groups = group_files(files, params=GroupingParams(min_files=2, max_files=4))root = next(g for g in groups if "setup" in " ".join(g.members) or "f00.py" in g.members)- assert root.target_path == ""- assert "#" not in root.target_path+ assert root.target_path == "root"+ assert all(g.target_path for g in groups)assert all("#" not in g.target_path for g in groups)+ def test_a_repository_with_its_own_root_directory_still_gets_unique_targets(self):+ """A top-level directory literally called ``root`` is not a conflict.++ The root group's name is picked against the directories that exist,+ so it steps aside rather than colliding — a collision here would mint+ one page id for two pages and silently drop one on persist.+ """+ files = _repo({"": 3, "root": 6, "src/pkg": 6})+ groups = group_files(files, params=GroupingParams(min_files=2, max_files=8))+ targets = [g.target_path for g in groups]++ assert len(targets) == len(set(targets)), targets+ assert "root" in targets+ assert any(t.startswith("_root") for t in targets), targets+def test_structural_key_follows_members_not_target(self):a = group_files(_repo({"src/one": 4}), params=TINY)[0]b = group_files(_repo({"src/two": 4}), params=TINY)[0]@@ -341,7 +359,5 @@ class TestLayers:files = _repo({"src/x": 4})layers = {f: ("layer:a" if i % 2 else "layer:b") for i, f in enumerate(files)}first = group_files(files, layer_of_file=layers, params=TINY)[0]- second = group_files(- list(reversed(files)), layer_of_file=layers, params=TINY- )[0]+ second = group_files(list(reversed(files)), layer_of_file=layers, params=TINY)[0]assert first.dominant_layer == second.dominant_layerdiff --git a/tests/unit/generation/test_concept_outline.py b/tests/unit/generation/test_concept_outline.pyindex 35ad33e9e..f25dafb29 100644--- a/tests/unit/generation/test_concept_outline.py+++ b/tests/unit/generation/test_concept_outline.py@@ -121,8 +121,7 @@ class TestModelCannotBreakStructure:groups = group_files(FILES, params=TINY)_payload, index = build_payload(groups)named, _ = decode_response(- {"sections": [{"title": "Core", "groups": [{"a": 1}, ["b"], None]}],- "names": {}},+ {"sections": [{"title": "Core", "groups": [{"a": 1}, ["b"], None]}], "names": {}},index,)assert len(named) == len(index)@@ -142,8 +141,7 @@ class TestModelCannotBreakStructure:@pytest.mark.asyncioasync def test_junk_types_in_the_response_are_ignored(self):content = (- '{"sections": {"not": "a list"},'- ' "names": {"g01": ["not", "a", "dict"], "g02": 7}}'+ '{"sections": {"not": "a list"}, "names": {"g01": ["not", "a", "dict"], "g02": 7}}')outline, report = await plan_outline(_inputs(), provider=FakeProvider(content), params=TINY, repair=False@@ -235,8 +233,7 @@ class TestValidator:ConceptSection(title="All",pages=[- ConceptPage(title=f"Page {i}", scope="s", group=g)- for i, g in enumerate(groups)+ ConceptPage(title=f"Page {i}", scope="s", group=g) for i, g in enumerate(groups)],))@@ -324,8 +321,7 @@ class TestPayload:from repowise.core.generation.concept_tree.grouping import ConceptGroupgroup = ConceptGroup(- members=[f"src/aaa/a{i}.py" for i in range(6)]- + [f"src/zzz/z{i}.py" for i in range(6)],+ members=[f"src/aaa/a{i}.py" for i in range(6)] + [f"src/zzz/z{i}.py" for i in range(6)],dirs=["src/aaa", "src/zzz"],target_path="src/aaa",)@@ -344,3 +340,43 @@ class TestPayload:)payload, _ = build_payload([group])assert sorted(payload["groups"][0]["subdirs"]) == ["git_indexer", "graph"]+++def test_layer_prefixed_title_does_not_repeat_the_layer_word():+ """A ``docs`` group in a "Docs & Tooling" layer is not "Docs Tooling Docs".++ The layer prefix exists to disambiguate a one-word directory name. When+ the label already contains that word the prefix adds nothing and the+ repetition reads as a generation bug.+ """+ from repowise.core.generation.concept_tree.grouping import ConceptGroup+ from repowise.core.generation.concept_tree.naming import deterministic_title++ group = ConceptGroup(+ members=["docs/conf.py", "docs/build.py"],+ dirs=["docs"],+ target_path="docs",+ )+ title = deterministic_title(group, "Docs Tooling")++ assert title == "Docs Tooling"+ assert title.lower().split().count("docs") == 1+++def test_a_trailing_number_stays_with_its_word():+ """``c4`` is the C4 model, not a C and a 4.++ The camel-case splitter treats a digit run as its own token, which is+ right for ``v2Api`` and wrong for every directory whose name ends in a+ number. Titles are the one place that leak is visible.+ """+ from repowise.core.generation.concept_tree.grouping import ConceptGroup+ from repowise.core.generation.concept_tree.naming import deterministic_title++ group = ConceptGroup(+ members=["src/ui/c4/nodes/a.ts", "src/ui/c4/nodes/b.ts"],+ dirs=["src/ui/c4/nodes"],+ target_path="src/ui/c4/nodes",+ )++ assert deterministic_title(group) == "C4 Nodes"diff --git a/tests/unit/generation/test_deterministic_templates.py b/tests/unit/generation/test_deterministic_templates.pyindex 86d1cdbf5..1591bfad5 100644--- a/tests/unit/generation/test_deterministic_templates.py+++ b/tests/unit/generation/test_deterministic_templates.py@@ -374,3 +374,67 @@ def test_as_markdown_is_idempotent():):once = as_markdown(src)assert as_markdown(once) == once+++# ---------------------------------------------------------------------------+# The concept page's own header+# ---------------------------------------------------------------------------+++def _module_ctx(assembler, files):+ from repowise.core.generation.context_assembler import FilePageContext++ contexts = [+ FilePageContext(+ file_path=f,+ language="python",+ symbols=[],+ docstring=None,+ imports=[],+ exports=[],+ dependencies=[],+ dependents=[],+ pagerank_score=0.1,+ betweenness_score=0.0,+ is_entry_point=False,+ file_source_snippet="",+ community_id=None,+ is_api_contract=False,+ is_test=False,+ parse_errors=[],+ estimated_tokens=10,+ )+ for f in files+ ]+ return assembler.assemble_module_page("Ingestion Pipeline", "python", contexts, None)+++def test_module_page_leads_with_the_concept_title(generator):+ """The H1 is the group's name, and the directories sit under it.++ A concept page covers several directories, so the old ``# Module: <path>``+ heading had nothing honest to put after the colon. The title says what the+ group is; the line beneath says where to go and look.+ """+ ctx = _module_ctx(generator._assembler, ["src/ingest/read.py", "src/parse/ast.py"])+ page = generator._deterministic_module_page(ctx, "src/ingest", "Ingestion Pipeline", None)++ assert page.title == "Ingestion Pipeline"+ assert page.content.startswith("# Ingestion Pipeline\n")+ assert "Module:" not in page.content+ assert "`src/ingest`" in page.content and "`src/parse`" in page.content+++def test_module_page_of_root_files_says_so(generator):+ """Files at the repository root have no directory to name.++ ``PurePosixPath("setup.py").parent`` is ``"."``, and printing a bare dot+ where a path belongs reads as a bug. Root-level files are a real case:+ the grouper anchors a group there whenever a repository keeps code at the+ top level.+ """+ ctx = _module_ctx(generator._assembler, ["setup.py", "main.py"])+ page = generator._deterministic_module_page(ctx, "root", "Project Entry Points", None)++ assert "Repository root" in page.content+ assert "`.`" not in page.contentdiff --git a/tests/unit/generation/test_models.py b/tests/unit/generation/test_models.pyindex 0f6347185..a87c27026 100644--- a/tests/unit/generation/test_models.py+++ b/tests/unit/generation/test_models.py@@ -153,8 +153,6 @@ def test_generation_config_defaults():assert config.staleness_threshold_days == 7assert config.expiry_threshold_days == 30assert config.top_symbol_percentile == 0.20- assert config.module_grouping == "curated"- assert config.min_module_size == 3assert config.large_file_source_pct == 0.4assert config.reasoning == "auto"diff --git a/tests/unit/generation/test_page_generator.py b/tests/unit/generation/test_page_generator.pyindex d39dd6b7e..d0a8150af 100644--- a/tests/unit/generation/test_page_generator.py+++ b/tests/unit/generation/test_page_generator.py@@ -473,10 +473,7 @@ async def test_generate_all_uses_in_memory_kg_modules_without_artifact_file():max_tokens=256,token_budget=100_000,max_concurrency=2,- module_grouping="curated",- min_module_size=2,coverage_pct=1.0,- module_page_share=1.0,dedupe_near_clones=False, # synthetic files are identical by design)provider = MockProvider()diff --git a/tests/unit/generation/test_page_selection.py b/tests/unit/generation/test_page_selection.pyindex 15e2efea3..db29025d6 100644--- a/tests/unit/generation/test_page_selection.py+++ b/tests/unit/generation/test_page_selection.py@@ -84,37 +84,12 @@ def test_windows_target_paths_normalize() -> None:# ----------------------------------------------------------------------------# Test directories do not get a module page+# Test files do not enter the concept tree# ------------------------------------------------------------------------------def _pf(path, is_test):- from types import SimpleNamespace-- return SimpleNamespace(file_info=SimpleNamespace(path=path, is_test=is_test))---def test_a_test_directory_is_not_a_module():- from repowise.core.generation.selection.selector import _is_test_group-- assert _is_test_group([_pf("tests/test_a.py", True), _pf("tests/test_b.py", True)])---def test_a_production_module_with_a_colocated_test_keeps_its_page():- """Go and Jest put the test beside the source; that is still a module."""- from repowise.core.generation.selection.selector import _is_test_group-- files = [_pf("pkg/a.go", False), _pf("pkg/b.go", False), _pf("pkg/a_test.go", True)]- assert not _is_test_group(files)---def test_an_even_split_is_not_a_test_group():- from repowise.core.generation.selection.selector import _is_test_group-- assert not _is_test_group([_pf("pkg/a.py", False), _pf("pkg/a_test.py", True)])---def test_empty_group_is_not_a_test_group():- from repowise.core.generation.selection.selector import _is_test_group-- assert not _is_test_group([])+#+# The majority-vote group filter that used to live here is gone with the+# per-directory grouping. Test files are now excluded before grouping runs,+# one file at a time, so a production directory with a colocated test keeps+# its page and the test file simply is not a member of it. That is asserted+# against the real selector in+# ``test_selection_concept_groups.py::test_test_files_never_enter_the_concept_tree``.diff --git a/tests/unit/generation/test_selection_budget.py b/tests/unit/generation/test_selection_budget.pyindex 544c0c5d4..ccf5e5df6 100644--- a/tests/unit/generation/test_selection_budget.py+++ b/tests/unit/generation/test_selection_budget.py@@ -9,7 +9,6 @@ configured ``coverage_pct``. They are the contract that prevents thefrom __future__ import annotationsfrom dataclasses import dataclass, field-from unittest.mock import MagicMockimport pytest@@ -24,7 +23,6 @@ from repowise.core.generation.selection.budget import (compute_budget,)-# ---------------------------------------------------------------------------# Lightweight ParsedFile / Symbol stand-ins# ---------------------------------------------------------------------------@@ -179,7 +177,14 @@ def test_allocate_budget_respects_candidate_supply():],)def test_select_pages_budget_tracks_coverage(coverage_pct, expected_total):- """Total page count tracks coverage_pct within ±10%."""+ """Budgeted page count tracks coverage_pct.++ ``module_page`` is excluded from the sum on purpose. The concept groups+ are a total partition of the production files, so that bucket is taken+ whole rather than rationed; folding it in would assert the opposite of the+ contract it now has. Its own invariant is asserted separately by+ :func:`test_module_bucket_is_not_rationed_by_coverage`.+ """parsed, pagerank, betweenness, community = _build_synthetic_repo(958)cfg = GenerationConfig(coverage_pct=coverage_pct)inputs = SelectionInputs(@@ -194,12 +199,58 @@ def test_select_pages_budget_tracks_coverage(coverage_pct, expected_total):)selection = select_pages(inputs)- total = sum(selection.counts().values())+ counts = selection.counts()+ total = sum(v for k, v in counts.items() if k != "module_page")tolerance = max(20, int(expected_total * 0.20))- assert abs(total - expected_total) <= tolerance, (- f"coverage={coverage_pct}: expected ~{expected_total}, got {total}"- )+ assert (+ abs(total - expected_total) <= tolerance+ ), f"coverage={coverage_pct}: expected ~{expected_total}, got {total}"+++def test_module_bucket_is_not_rationed_by_coverage():+ """Every concept group is emitted, whatever the coverage setting.++ The partition is total, so a rationed module bucket does not make a+ smaller wiki — it makes one with production files that belong to no page,+ and the tree stops being a map of the repository. Asserted across the+ coverage range because the budget is what used to truncate it.+ """+ parsed, pagerank, betweenness, community = _build_synthetic_repo(958)++ def _selection_for(pct: float):+ return select_pages(+ SelectionInputs(+ parsed_files=parsed,+ pagerank=pagerank,+ betweenness=betweenness,+ community=community,+ community_info=None,+ sccs=[],+ git_meta_map=None,+ config=GenerationConfig(coverage_pct=pct),+ )+ )++ selections = {pct: _selection_for(pct) for pct in (0.10, 0.20, 0.50)}+ group_counts = {pct: len(s.module_groups) for pct, s in selections.items()}++ assert (+ len(set(group_counts.values())) == 1+ ), f"coverage changed the concept page count: {group_counts}"+ assert all(c > 0 for c in group_counts.values())++ # The allocation must agree with what was emitted, or the cost estimator+ # quotes a number the run does not honour.+ for pct, sel in selections.items():+ assert sel.allocation is not None+ assert sel.allocation.module_page == len(sel.module_groups), pct++ # Totality: every production file the grouper was given lands on exactly+ # one page. This is the property rationing used to break.+ for pct, sel in selections.items():+ claimed = [p for g in sel.module_groups for p in g.file_paths]+ assert len(claimed) == len(set(claimed)), f"double-claimed at {pct}"def test_select_pages_emits_no_pages_for_empty_repo():@@ -283,15 +334,20 @@ def test_landmarks_are_pulled_into_budget_without_inflating_count():def test_landmarks_respect_coverage_across_sizes():- """The total page count still tracks coverage_pct with landmarks active."""+ """The budgeted page count still tracks coverage_pct with landmarks active.++ ``module_page`` is excluded for the same reason as+ :func:`test_select_pages_budget_tracks_coverage`: the concept bucket is+ taken whole, so it does not answer to the coverage budget.+ """for n, pct in [(40, 0.10), (500, 0.20), (1200, 0.10)]:parsed, pr, bet, comm = _build_synthetic_repo(n, entry_points=5)sel = _select(parsed, pr, bet, comm, pct)- total = sum(sel.counts().values())+ total = sum(v for k, v in sel.counts().items() if k != "module_page")# Landmark pull-in displaces rather than adds, so the total stays in# the same ±20% band the budget contract guarantees.expected = compute_budget(n, pct)tolerance = max(20, int(expected * 0.30))- assert abs(total - expected) <= tolerance, (- f"n={n} pct={pct}: expected ~{expected}, got {total}"- )+ assert (+ abs(total - expected) <= tolerance+ ), f"n={n} pct={pct}: expected ~{expected}, got {total}"diff --git a/tests/unit/generation/test_selection_concept_groups.py b/tests/unit/generation/test_selection_concept_groups.pynew file mode 100644index 000000000..b1c197c0c--- /dev/null+++ b/tests/unit/generation/test_selection_concept_groups.py@@ -0,0 +1,309 @@+"""Concept grouping in the selection layer.++Replaces the curated/community/top-dir grouping tests. That grouping is gone,+not deprecated: ``module_page`` is now produced from the concept partition+(``concept_tree.grouping.group_files``), so the properties worth asserting+changed from "which of three sources won" to the two the partition+guarantees — every production file is covered exactly once, and the same+input always produces the same page identities.++Fixtures deliberately disagree with the alphabet where ordering matters, and+every path fixture includes a root-level file, because the repository root's+directory path is the empty string and a falsy check cannot tell it from "no+directory found". That defect shipped once.+"""++from __future__ import annotations++import os+import subprocess+import sys+import textwrap+from pathlib import Path++from repowise.core.generation.models import GenerationConfig+from repowise.core.generation.selection import SelectionInputs, select_pages+from repowise.core.generation.selection.selector import _build_module_groups+from tests.unit.generation.test_selection_budget import (+ FakeFileInfo,+ FakeParsedFile,+ FakeSymbol,+)++_REPO_ROOT = Path(__file__).resolve().parents[3]++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def _paths() -> list[str]:+ """Production paths spanning several subtrees, plus root-level files.++ PageRank below is assigned so the highest-scoring group is NOT the one+ that sorts first alphabetically, so an ordering assertion cannot pass by+ accident on a sorted-by-path implementation.+ """+ return (+ [f"packages/app/src/core/ingestion/f{i}.py" for i in range(8)]+ + [f"packages/app/src/core/analysis/f{i}.py" for i in range(6)]+ + [f"packages/app/src/ui/c4/f{i}.py" for i in range(5)]+ + ["packages/app/src/ui/tiny/f0.py", "packages/app/src/ui/tiny/f1.py"]+ # Root-level files: the empty-string directory is real.+ + ["setup.py", "main.py"]+ )+++def _test_paths() -> list[str]:+ return [f"tests/unit/test_f{i}.py" for i in range(6)]+++def _inputs(+ *,+ with_tests: bool = True,+ kg_modules: list[dict] | None = None,+ paths: list[str] | None = None,+):+ prod = paths if paths is not None else _paths()+ paths = prod + (_test_paths() if with_tests else [])+ parsed = [+ FakeParsedFile(+ file_info=FakeFileInfo(path=p, is_test=p.startswith("tests/")),+ symbols=[FakeSymbol(name="fn")],+ )+ for p in paths+ ]+ # ui/c4 carries the highest mass, and "packages/app/src/ui/c4" sorts after+ # "packages/app/src/core/...", so score-descending order must not equal+ # path order.+ pagerank = {p: (1.0 if "/ui/c4/" in p else 0.1) for p in paths}+ return SelectionInputs(+ parsed_files=parsed,+ pagerank=pagerank,+ betweenness={p: 0.0 for p in paths},+ community={p: 0 for p in paths},+ community_info=None,+ sccs=[],+ git_meta_map=None,+ config=GenerationConfig(coverage_pct=0.20),+ kg_modules=kg_modules,+ )+++# ---------------------------------------------------------------------------+# Coverage and totality+# ---------------------------------------------------------------------------+++def test_every_production_file_is_covered_exactly_once():+ groups = [g for _, g in _build_module_groups(_inputs())]+ claimed = [p for g in groups for p in g.file_paths]++ assert sorted(claimed) == sorted(_paths()), "the partition is not total"+ assert len(claimed) == len(set(claimed)), "a file was claimed by two pages"+++def test_root_level_files_get_a_usable_target_path():+ """The repository root is a real directory whose path is ``""``.++ A group anchored there must not persist an empty ``target_path``: the page+ id is ``"{type}:{target_path}"``, so an empty target mints ``module_page:``+ and leaves the page with nothing for the tree, the breadcrumbs or the+ bench gold set to match on.+ """+ groups = [g for _, g in _build_module_groups(_inputs())]+ owning = [g for g in groups if "setup.py" in g.file_paths]++ assert owning, "root-level files were dropped from the partition"+ assert any("main.py" in g.file_paths for g in groups)+ for g in groups:+ assert g.key, "a group persisted an empty target_path"+ assert any(+ g.key == "root" for g in owning+ ), f"root-anchored group did not get the root target: {[g.key for g in owning]}"+++def test_test_files_never_enter_the_concept_tree():+ """D8: tests keep file pages, they do not get concept pages."""+ groups = [g for _, g in _build_module_groups(_inputs())]+ claimed = {p for g in groups for p in g.file_paths}++ assert not any(p.startswith("tests/") for p in claimed)+ # And their absence is not an artifact of them being absent from input.+ assert any(p.file_info.path.startswith("tests/") for p in _inputs().parsed_files)+++def test_dropping_the_test_files_does_not_change_the_partition():+ """Excluding tests must be a filter, not a force that reshapes groups."""+ with_tests = [g.structural_key for _, g in _build_module_groups(_inputs())]+ without = [g.structural_key for _, g in _build_module_groups(_inputs(with_tests=False))]++ assert with_tests == without+++# ---------------------------------------------------------------------------+# Identity+# ---------------------------------------------------------------------------+++def test_groups_carry_a_concept_prefixed_structural_key():+ """The prefix is what lets a wiki tell a concept page from an old one."""+ groups = [g for _, g in _build_module_groups(_inputs())]++ assert groups+ assert all(g.structural_key.startswith("concept-") for g in groups)+ keys = [g.structural_key for g in groups]+ assert len(keys) == len(set(keys)), "two groups share an identity"+++def test_target_paths_are_unique():+ """The page id is derived from the target path alone.++ Two groups resolving to one target is a row collision: the second page+ silently overwrites the first on persist.+ """+ groups = [g for _, g in _build_module_groups(_inputs())]+ keys = [g.key for g in groups]++ assert len(keys) == len(set(keys)), f"duplicate target paths: {keys}"+++def test_identity_is_stable_across_processes():+ """Hash randomisation must not move a page's identity.++ Run in two subprocesses under different ``PYTHONHASHSEED`` values. Calling+ the function twice in one process cannot fail this way and therefore+ cannot test it.+ """+ script = textwrap.dedent(+ """+ from tests.unit.generation.test_selection_concept_groups import (+ _build_module_groups, _inputs,+ )+ groups = [g for _, g in _build_module_groups(_inputs())]+ print("|".join(f"{g.key}={g.structural_key}" for g in groups))+ """+ )+ outs = []+ for seed in ("0", "12345"):+ env = {**os.environ, "PYTHONHASHSEED": seed}+ proc = subprocess.run(+ [sys.executable, "-c", script],+ capture_output=True,+ text=True,+ env=env,+ cwd=str(_REPO_ROOT),+ timeout=120,+ )+ assert proc.returncode == 0, proc.stderr+ outs.append(proc.stdout.strip())++ assert outs[0] == outs[1], "page identity moved with the hash seed"+ assert outs[0], "the subprocess produced no groups, so this asserted nothing"+++# ---------------------------------------------------------------------------+# Ordering and naming+# ---------------------------------------------------------------------------+++def test_ranked_by_summed_pagerank_not_by_path():+ """The heaviest subsystem sorts first even though its path sorts last."""+ scored = _build_module_groups(_inputs())+ ordered_keys = [g.key for _, g in scored]++ assert len(scored) > 1, "a one-group fixture cannot test ordering"+ # The fixture is only meaningful if path order disagrees with score order,+ # or the assertion below would pass on a sorted-by-path implementation.+ assert ordered_keys != sorted(+ ordered_keys+ ), f"fixture is degenerate: score order equals path order ({ordered_keys})"+ # ui/c4 carries 1.0 per file against 0.1 elsewhere, so whichever group+ # holds it must come first.+ top_group = scored[0][1]+ assert any(+ "/ui/c4/" in p for p in top_group.file_paths+ ), f"expected the ui/c4 mass to rank first, got {top_group.key}"+++def test_display_is_a_name_not_a_bare_path():+ groups = [g for _, g in _build_module_groups(_inputs())]++ for g in groups:+ assert g.display, f"{g.key} has no title"+ assert "/" not in g.display, f"{g.display} is a path, not a name"+++def test_layer_labels_from_the_kg_reach_the_title():+ """The KG layer map is read when present and absent-safe when not."""+ modules = [+ {+ "id": "module:ui-c4",+ "path": "packages/app/src/ui/c4",+ "layerId": "layer:presentation",+ "nodeIds": [f"file:{p}" for p in _paths() if "/ui/c4/" in p],+ }+ ]+ with_kg = [g for _, g in _build_module_groups(_inputs(kg_modules=modules))]+ without_kg = [g for _, g in _build_module_groups(_inputs())]++ # Membership is unchanged by the layer signal: it steers merging of+ # adjacent runs, it never forces a split.+ assert [g.structural_key for g in with_kg] == [g.structural_key for g in without_kg]+ assert all(g.display for g in with_kg)++ # The label itself has to reach a title, or the map is being read and+ # thrown away. Titles differ only where the layer applies: the prefix+ # exists to rescue a name too short to say anything, so groups whose+ # paths already yield two words are identical either way.+ titled = [g.display for g in with_kg]+ untitled = [g.display for g in without_kg]+ assert titled != untitled, titled+ assert any("Presentation" in t for t in titled), titled+++# ---------------------------------------------------------------------------+# The bucket is not rationed+# ---------------------------------------------------------------------------+++def test_select_pages_emits_every_group():+ inputs = _inputs()+ groups = _build_module_groups(inputs)+ selection = select_pages(inputs)++ assert len(selection.module_groups) == len(groups)+ assert selection.allocation is not None+ assert selection.allocation.module_page == len(groups)+++def test_structural_key_survives_selection():+ """The key the grouper minted has to reach the page, not be recomputed."""+ selection = select_pages(_inputs())++ assert selection.module_groups+ assert all(g.structural_key.startswith("concept-") for g in selection.module_groups)+++# ---------------------------------------------------------------------------+# Titles a reader can tell apart+# ---------------------------------------------------------------------------+++def test_two_groups_never_share_a_title():+ """Same directory name in two packages must not produce two identical rows.++ Page identity is structural, so the pages stay distinct — but the tree+ shows the title, and two rows reading "Ui Components" are the same row as+ far as a reader is concerned.+ """+ paths = (+ [f"packages/alpha/src/ui/f{i}.py" for i in range(8)]+ + [f"packages/beta/src/ui/f{i}.py" for i in range(8)]+ + ["setup.py"]+ )+ groups = [g for _, g in _build_module_groups(_inputs(paths=paths))]++ titles = [g.display for g in groups]+ assert len(titles) == len(set(titles)), titles+ assert len({g.key for g in groups}) == len(groups)diff --git a/tests/unit/generation/test_selection_curated_modules.py b/tests/unit/generation/test_selection_curated_modules.pydeleted file mode 100644index 43ff6c195..000000000--- a/tests/unit/generation/test_selection_curated_modules.py+++ /dev/null@@ -1,278 +0,0 @@-"""Curated module grouping in the selection layer.--Covers the ``module_grouping="curated"`` source (key = real dir path,-display = curated name, cohesion = None, Σ-PageRank ranking), every row of-the fallback matrix, and the golden inertness contract: with the default-``"community"`` grouping, passing ``kg_modules`` changes NOTHING — the-ship-dark guarantee that every existing path stays byte-identical.-"""--from __future__ import annotations--from repowise.core.generation.models import GenerationConfig-from repowise.core.generation.selection import SelectionInputs, select_pages-from repowise.core.generation.selection.selector import _build_module_groups-from tests.unit.generation.test_selection_budget import (- FakeFileInfo,- FakeParsedFile,- FakeSymbol,- _build_synthetic_repo,-)--# ----------------------------------------------------------------------------# Fixtures-# ------------------------------------------------------------------------------def _repo_with_modules():- """A synthetic repo plus a matching curated-modules artifact."""- paths = (- [f"packages/app/src/core/ingestion/f{i}.py" for i in range(8)]- + [f"packages/app/src/core/analysis/f{i}.py" for i in range(6)]- + [f"packages/app/src/ui/c4/f{i}.py" for i in range(5)]- + ["packages/app/src/ui/tiny/f0.py", "packages/app/src/ui/tiny/f1.py"]- )- parsed = [- FakeParsedFile(file_info=FakeFileInfo(path=p), symbols=[FakeSymbol(name="fn")])- for p in paths- ]- # ui/c4 members carry the highest PageRank so Σ-PageRank must rank that- # module first despite it being smaller than core/ingestion.- pagerank = {p: (1.0 if "/ui/c4/" in p else 0.1) for p in paths}- community = {p: (0 if "/core/" in p else 1) for p in paths}-- modules = [- {- "id": "module:packages-app-src-core-ingestion",- "name": "core/ingestion",- "path": "packages/app/src/core/ingestion",- "layerId": "layer:service",- "nodeIds": [f"file:{p}" for p in paths if "/core/ingestion/" in p],- "language": "python",- },- {- "id": "module:packages-app-src-core-analysis",- "name": "core/analysis",- "path": "packages/app/src/core/analysis",- "layerId": "layer:service",- "nodeIds": [f"file:{p}" for p in paths if "/core/analysis/" in p],- "language": "python",- },- {- "id": "module:packages-app-src-ui-c4",- "name": "ui/c4",- "path": "packages/app/src/ui/c4",- "layerId": "layer:ui",- "nodeIds": [f"file:{p}" for p in paths if "/ui/c4/" in p],- "language": "python",- },- {- # Below min_module_size=3 → must be skipped, never a page.- "id": "module:packages-app-src-ui-tiny",- "name": "ui/tiny",- "path": "packages/app/src/ui/tiny",- "layerId": "layer:ui",- "nodeIds": [f"file:{p}" for p in paths if "/ui/tiny/" in p],- "language": "python",- },- ]- return parsed, pagerank, community, modules---def _inputs(parsed, pagerank, community, *, cfg=None, kg_modules=None):- return SelectionInputs(- parsed_files=parsed,- pagerank=pagerank,- betweenness=dict.fromkeys(pagerank, 0.0),- community=community,- community_info=None,- sccs=[],- git_meta_map=None,- config=cfg or GenerationConfig(),- kg_modules=kg_modules,- )---def _cfg(grouping: str) -> GenerationConfig:- return GenerationConfig(module_grouping=grouping)---# ----------------------------------------------------------------------------# Curated grouping source-# ------------------------------------------------------------------------------class TestCuratedGrouping:- def test_groups_keyed_by_dir_path_with_curated_names(self):- parsed, pr, comm, modules = _repo_with_modules()- groups = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)- )- by_key = {g.key: g for _, g in groups}- assert "packages/app/src/core/ingestion" in by_key- g = by_key["packages/app/src/core/ingestion"]- assert g.display == "core/ingestion"- assert g.label == "core/ingestion"- assert g.cohesion is None- assert g.language == "python"- assert len(g.file_paths) == 8-- def test_ranked_by_sum_pagerank_of_members(self):- parsed, pr, comm, modules = _repo_with_modules()- groups = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)- )- # ui/c4: 5 x 1.0 = 5.0 beats core/ingestion: 8 x 0.1 = 0.8.- assert groups[0][1].display == "ui/c4"- scores = [s for s, _ in groups]- assert scores == sorted(scores, reverse=True)-- def test_min_module_size_floor_kills_tiny_modules(self):- parsed, pr, comm, modules = _repo_with_modules()- groups = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)- )- assert "packages/app/src/ui/tiny" not in {g.key for _, g in groups}-- def test_members_restricted_to_parsed_code_files(self):- parsed, pr, comm, modules = _repo_with_modules()- modules[0]["nodeIds"].append("file:packages/app/src/core/ingestion/ghost.py")- groups = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)- )- by_key = {g.key: g for _, g in groups}- assert not any(- "ghost" in p- for p in by_key["packages/app/src/core/ingestion"].file_paths- )-- def test_no_community_ids_in_curated_keys(self):- parsed, pr, comm, modules = _repo_with_modules()- groups = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)- )- assert not any(g.key.startswith("community-") for _, g in groups)---# ----------------------------------------------------------------------------# Fallback matrix (one test per row)-# ------------------------------------------------------------------------------class TestFallbackMatrix:- def test_curated_without_artifact_falls_back_to_community(self):- """curation off / degraded / no KG json → today's community path."""- parsed, pr, _bet, comm = _build_synthetic_repo(100)- curated_no_artifact = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=None)- )- community = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("community"))- )- assert curated_no_artifact == community-- def test_curated_with_all_modules_below_floor_emits_none_not_community(self):- """A present-but-filtered artifact must not mix vocabularies."""- parsed, pr, comm, modules = _repo_with_modules()- tiny_only = [m for m in modules if m["name"] == "ui/tiny"]- groups = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=tiny_only)- )- assert groups == []-- def test_explicit_community_honored_even_with_artifact(self):- parsed, pr, comm, modules = _repo_with_modules()- with_artifact = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("community"), kg_modules=modules)- )- without = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("community"))- )- assert with_artifact == without- assert not any(g.key.startswith("packages/") for _, g in with_artifact)-- def test_explicit_top_dir_honored_even_with_artifact(self):- parsed, pr, comm, modules = _repo_with_modules()- with_artifact = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("top_dir"), kg_modules=modules)- )- without = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("top_dir"))- )- assert with_artifact == without- assert {g.key for _, g in with_artifact} == {"packages"}---# ----------------------------------------------------------------------------# Golden inertness — the ship-dark contract-# ------------------------------------------------------------------------------class TestGoldenByteIdentity:- def test_community_selection_identical_with_and_without_kg_modules(self):- """With the "community" escape hatch, kg_modules must change NOTHING.-- This was the ship-dark rollout guarantee while "community" was the- default; post-flip it pins the revert story — setting- module_grouping="community" restores the pre-curation selection- byte-for-byte regardless of what the KG artifact carries.- """- parsed, pr, bet, comm = _build_synthetic_repo(300)- _, _, _, modules = _repo_with_modules()- cfg = GenerationConfig(module_grouping="community")-- def run(kg_modules):- return select_pages(- SelectionInputs(- parsed_files=parsed,- pagerank=pr,- betweenness=bet,- community=comm,- community_info=None,- sccs=[],- git_meta_map=None,- config=cfg,- kg_modules=kg_modules,- )- )-- assert run(None) == run(modules)-- def test_curated_selection_is_deterministic(self):- parsed, pr, comm, modules = _repo_with_modules()- cfg = _cfg("curated")-- def run():- return select_pages(- _inputs(parsed, pr, comm, cfg=cfg, kg_modules=modules)- )-- a, b = run(), run()- assert a.module_groups == b.module_groups- assert a == b---class TestWholeLayerDedupe:- """Modules 1:1 with a layer never get a module page (layer_page covers them)."""-- def test_whole_layer_module_skipped(self):- parsed, pr, comm, modules = _repo_with_modules()- modules = [dict(m) for m in modules]- modules[0]["wholeLayer"] = True # core/ingestion now 1:1 with its layer- groups = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)- )- keys = {g.key for _, g in groups}- assert "packages/app/src/core/ingestion" not in keys- assert "packages/app/src/ui/c4" in keys-- def test_all_whole_layer_yields_no_module_pages_not_community(self):- # Flat-lib shape: every module is 1:1 with a layer → zero module- # pages, and NO fallback to community grouping (vocabulary stays- # curated; the layer pages carry the documentation).- parsed, pr, comm, modules = _repo_with_modules()- modules = [dict(m, wholeLayer=True) for m in modules]- groups = _build_module_groups(- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)- )- assert groups == []diff --git a/tests/unit/generation/test_templates.py b/tests/unit/generation/test_templates.pyindex 6916b0b87..31ad63c29 100644--- a/tests/unit/generation/test_templates.py+++ b/tests/unit/generation/test_templates.py@@ -114,7 +114,7 @@ def test_file_page_contains_file_path(jinja_env, file_page_ctx):@pytest.fixture(scope="module")def module_page_ctx() -> ModulePageContext:return ModulePageContext(- module_path="python_pkg",+ title="Calculation Engine",language="python",total_symbols=5,public_symbols=3,@@ -123,6 +123,7 @@ def module_page_ctx() -> ModulePageContext:dependents=[],pagerank_mean=0.3,files=["python_pkg/calculator.py", "python_pkg/models.py"],+ directories=["python_pkg"],)@@ -136,9 +137,16 @@ def test_module_page_has_heading(jinja_env, module_page_ctx):assert "##" in result-def test_module_page_contains_module_path(jinja_env, module_page_ctx):+def test_module_page_names_the_group_and_where_it_lives(jinja_env, module_page_ctx):+ """The title is prose and the directories are the path. Both must appear.++ A concept page spans directories, so a reader who only gets the title has+ no way to go and look, and one who only gets a path has to guess what the+ group is for.+ """result = render(jinja_env, "module_page.j2", module_page_ctx)- assert module_page_ctx.module_path in result+ assert module_page_ctx.title in result+ assert "python_pkg" in result# ---------------------------------------------------------------------------@@ -339,5 +347,3 @@ def test_scc_page_has_heading(jinja_env, scc_page_ctx):def test_scc_page_contains_cycle_description(jinja_env, scc_page_ctx):result = render(jinja_env, "scc_page.j2", scc_page_ctx)assert scc_page_ctx.cycle_description in result--diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.pyindex 6d8fc788d..4bbc2e6dc 100644--- a/tests/unit/ingestion/test_traverser.py+++ b/tests/unit/ingestion/test_traverser.py@@ -781,3 +781,56 @@ class TestEntryPointFlag:flagged = self._flagged(tmp_path)assert "cli.py" in flaggedassert "pkg/cli.py" not in flagged+++# ---------------------------------------------------------------------------+# Test-file classification+# ---------------------------------------------------------------------------+++ "rel_path",+ [+ "tests/conftest.py",+ "tests/type_check/typing_app.py",+ "packages/ui/__tests__/brand.test.ts",+ "packages/ui/__tests__/chat/panel.tsx",+ "src/components/Button.test.tsx",+ "src/lib/parse.spec.ts",+ "test/fixtures/app.py",+ "spec/support/helper.rb",+ "src/pkg/tests/helpers.py",+ "pkg/test_thing.py",+ "pkg/thing_test.go",+ ],+)+def test_test_files_are_recognised_including_root_level_suites(rel_path):+ """Directory and filename conventions across languages, not just Python.++ Two gaps, both of which let test files into the concept tree. ``"/tests/"+ in path`` needs a slash on both sides, so a root-level ``tests/`` — the+ usual layout — read as production code. And the stem rule knew only+ ``test_x`` / ``x_test``, so the whole JavaScript convention+ (``__tests__/`` directories, ``x.test.ts`` filenames) was invisible:+ measured on this repository, 122 test files survived into the production+ set and five concept pages were nothing but tests.+ """+ from repowise.core.ingestion.traverser import _is_test_file++ assert _is_test_file(rel_path, Path(rel_path).name)+++ "rel_path",+ [+ "src/latest/api.py",+ "src/contest/rules.py",+ "protests/model.py",+ "src/testing_utils.py",+ ],+)+def test_production_paths_that_merely_contain_the_word_are_not_tests(rel_path):+ """Segment match, not substring: ``contest`` and ``latest`` are not tests."""+ from repowise.core.ingestion.traverser import _is_test_file++ assert not _is_test_file(rel_path, Path(rel_path).name)diff --git a/tests/unit/pipeline/test_authoritative_page_types.py b/tests/unit/pipeline/test_authoritative_page_types.pynew file mode 100644index 000000000..c4937c701--- /dev/null+++ b/tests/unit/pipeline/test_authoritative_page_types.py@@ -0,0 +1,122 @@+"""What a generating run claims to have decided, and therefore what the sweep may retire.++``authoritative_page_types`` is the signal ``_sweep_stale_generated_pages``+reads before deleting prior rows of a page type the run did not reproduce. Get+it wrong in the permissive direction and a degraded run wipes good pages; get+it wrong in the strict direction and superseded pages accumulate forever.++``module_page`` is the interesting one. The concept grouping partitions every+production file, takes no optional input, and is not rationed by the coverage+budget, so a run that produced any concept page produced all of them. It used+to be declared authoritative only when a curated knowledge-graph artifact was+present, which left exactly the KG-less run, the one carrying the most stale+rows, unable to clean up after itself.++The claim is still made on evidence rather than on "generation ran", because+there are ways to emit zero concept pages that do not mean "there is nothing+to document" — ``file_pages_only`` returns before that level, and a provider+error on it is logged per page rather than raised.+"""++from __future__ import annotations++from repowise.core.pipeline import run_pipeline+from repowise.core.providers.llm.mock import MockProvider+++def _write_repo(root):+ root.mkdir()+ (root / "pkg").mkdir()+ (root / "pkg" / "main.py").write_text(+ "def hello() -> str:\n return 'hello'\n",+ encoding="utf-8",+ )+ (root / "pkg" / "util.py").write_text(+ "def helper(n: int) -> int:\n return n + 1\n",+ encoding="utf-8",+ )+ return root+++async def test_generating_run_is_authoritative_for_module_pages(tmp_path):+ repo_path = _write_repo(tmp_path / "repo")++ result = await run_pipeline(+ repo_path,+ generate_docs=True,+ llm_client=MockProvider(),+ concurrency=1,+ test_run=True,+ )++ assert result.generated_pages+ assert "module_page" in result.authoritative_page_types+++async def test_a_run_that_does_not_generate_claims_nothing(tmp_path):+ """No pages produced means no page type was decided.++ An index-only run has no opinion about which concept pages should exist,+ so letting it claim authority would let it retire every one of them.+ """+ repo_path = _write_repo(tmp_path / "repo")++ result = await run_pipeline(+ repo_path,+ generate_docs=False,+ concurrency=1,+ test_run=True,+ )++ assert not result.authoritative_page_types+++async def test_a_run_that_never_reaches_the_concept_level_claims_nothing(tmp_path):+ """``file_pages_only`` stops before concept pages exist.++ Claiming authority there would let a file-pages-only run delete every+ concept page in the store, which is the most destructive shape this+ signal has.+ """+ from repowise.core.generation.models import GenerationConfig++ repo_path = _write_repo(tmp_path / "repo")++ result = await run_pipeline(+ repo_path,+ generate_docs=True,+ llm_client=MockProvider(),+ concurrency=1,+ test_run=True,+ generation_config=GenerationConfig(file_pages_only=True),+ )++ assert result.generated_pages+ assert not any(p.page_type == "module_page" for p in result.generated_pages)+ assert "module_page" not in result.authoritative_page_types+++async def test_a_repository_with_no_production_code_still_claims(tmp_path):+ """Zero pages because there is nothing to group is the case the sweep wants.++ A repository whose code was deleted, or that is only tests, should have+ its concept pages retired rather than kept forever behind an+ "it produced nothing, so trust nothing" rule.+ """+ repo_path = tmp_path / "repo"+ repo_path.mkdir()+ (repo_path / "tests").mkdir()+ (repo_path / "tests" / "test_thing.py").write_text(+ "def test_x():\n assert True\n", encoding="utf-8"+ )++ result = await run_pipeline(+ repo_path,+ generate_docs=True,+ llm_client=MockProvider(),+ concurrency=1,+ test_run=True,+ )++ assert not any(p.page_type == "module_page" for p in (result.generated_pages or []))+ assert "module_page" in result.authoritative_page_types