Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): concept groups replace per-directory module pages (#1024)

Raghav Chamadiyaabdc3a3fe3f9
diff --git a/packages/core/src/repowise/core/cost_estimator/approx.py b/packages/core/src/repowise/core/cost_estimator/approx.py
index 9706a7627..2e38f8f97 100644
--- a/packages/core/src/repowise/core/cost_estimator/approx.py
+++ b/packages/core/src/repowise/core/cost_estimator/approx.py
@@ -16,12 +16,6 @@ from __future__ import annotations
from .types import PageTypePlan
-# Observed module-page ratio: curated/community grouping lands near one
-# module page per ~25 selected file pages, floored at 1 and capped so a
-# monorepo estimate does not balloon.
-_FILES_PER_MODULE = 25
-_MAX_MODULE_PAGES = 24
-
# The curated onboarding collection has 8 fixed slots; slots without
# enough signal are skipped, so this biases slightly high on tiny repos.
_ONBOARDING_SLOTS = 8
@@ -38,9 +32,27 @@ def approximate_generation_plan(
file_pages = round(file_count * max(0.0, min(coverage_pct, 1.0)))
if file_count > 0:
file_pages = max(1, file_pages)
- module_pages = (
- min(_MAX_MODULE_PAGES, max(1, file_pages // _FILES_PER_MODULE)) if file_pages else 0
- )
+ # Concept pages partition every production file, so they scale with the
+ # repository rather than with the budgeted file-page slice, and they are
+ # not capped: a cap would quote a number the run cannot honour. The size
+ # ladder the grouper actually uses gives the expected group size, so this
+ # estimate moves with the real bounds instead of a remembered ratio.
+ #
+ # The ladder is a step function and this is fed *documentable* files
+ # (tests and config included) while the grouper sees production code
+ # only, so near a band edge the two can sit in different bands. Taking
+ # the larger of this band and the one below keeps the error on the
+ # over-quote side, which is the only side that does not surprise someone
+ # with a bill.
+ module_pages = 0
+ if file_count:
+ from repowise.core.generation.concept_tree.grouping import params_for
+
+ estimates = [
+ max(1, round(file_count / ((bounds.min_files + bounds.max_files) / 2)))
+ for bounds in (params_for(file_count), params_for(int(file_count * 0.8)))
+ ]
+ module_pages = max(estimates)
plans = [
PageTypePlan(page_type="file_page", count=file_pages, level=0),
diff --git a/packages/core/src/repowise/core/cost_estimator/plans.py b/packages/core/src/repowise/core/cost_estimator/plans.py
index f6afc65b0..d3421e4e8 100644
--- a/packages/core/src/repowise/core/cost_estimator/plans.py
+++ b/packages/core/src/repowise/core/cost_estimator/plans.py
@@ -37,10 +37,10 @@ def build_generation_plan(
config:
GenerationConfig — its ``coverage_pct`` drives the budget.
kg_modules:
- Curated wiki modules from the KG artifact, when available. Must be
- passed for ``module_grouping="curated"`` estimates to match the
- actual run — without it the selector falls back to community
- grouping, exactly as generation itself would without an artifact.
+ Curated wiki modules from the KG artifact, when available. Only
+ steers which adjacent directories merge into a concept group, so its
+ absence changes which pages exist rather than how many, and the
+ estimate holds either way.
"""
files = parsed_files
if skip_tests:
diff --git a/packages/core/src/repowise/core/generation/concept_tree/grouping.py b/packages/core/src/repowise/core/generation/concept_tree/grouping.py
index d6c0b967e..8a5ba8ba0 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/grouping.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/grouping.py
@@ -116,9 +116,7 @@ class ConceptGroup:
def __post_init__(self) -> None:
if not self.structural_key:
- self.structural_key = member_structural_key(
- self.members, prefix=STRUCTURAL_KEY_PREFIX
- )
+ self.structural_key = member_structural_key(self.members, prefix=STRUCTURAL_KEY_PREFIX)
@property
def file_count(self) -> int:
@@ -309,9 +307,7 @@ class _Partitioner:
return []
-def _absorb_thin(
- groups: list[ConceptGroup], part: _Partitioner
-) -> list[ConceptGroup]:
+def _absorb_thin(groups: list[ConceptGroup], part: _Partitioner) -> list[ConceptGroup]:
"""Fold groups too small to deserve a page into their closest neighbour.
The recursion flushes a remainder wherever a subtree boundary falls, so a
@@ -382,6 +378,18 @@ def _assign_targets(groups: list[ConceptGroup]) -> None:
wrong is two pages silently sharing a row, so the loop below checks the
result and walks to the next candidate rather than trusting it.
"""
+ # The repository root's directory path is the empty string, which is a
+ # real directory but cannot be a page id: the id is ``"{type}:{path}"``,
+ # so an empty path mints ``module_page:`` and leaves the page with nothing
+ # for the tree, the breadcrumbs or a gold set to match on. It gets a name
+ # here, inside the function that owns uniqueness, rather than downstream —
+ # a caller remapping ``""`` to ``"root"`` afterwards would collide with a
+ # repository that has a top-level directory actually called ``root``.
+ all_dirs = {d for group in groups for d in group.dirs}
+ root_name = "root"
+ while root_name in all_dirs:
+ root_name = f"_{root_name}"
+
taken: set[str] = set()
# Largest first so the group with the strongest claim to a directory keeps
# the shortest name; ties break on the key so the order never depends on
@@ -392,6 +400,12 @@ def _assign_targets(groups: list[ConceptGroup]) -> None:
candidates.extend(d for d in group.dirs if d and d not in candidates)
if owned and owned not in candidates:
candidates.append(owned)
+ # The root is substituted before the uniqueness check, never after.
+ # Two groups can both reduce to it — one holding the root-level files,
+ # another spanning siblings whose only common ancestor is the root —
+ # and renaming afterwards would let the second reuse a name the first
+ # already holds.
+ candidates = [root_name if c == "" else c for c in candidates]
# ``None`` rather than "" for "nothing chosen yet": the repository root
# is a real, legitimate directory whose path *is* the empty string, and
# a falsy check cannot tell it apart from failure. It could not, once —
@@ -407,7 +421,8 @@ def _assign_targets(groups: list[ConceptGroup]) -> None:
# Every real directory is spoken for. Fall back to the members'
# common prefix qualified by the group's identity, which is stable
# and unique even though it is no longer only a directory.
- chosen = f"{owned}#{group.structural_key.rsplit('-', 1)[-1]}"
+ prefix = owned or root_name
+ chosen = f"{prefix}#{group.structural_key.rsplit('-', 1)[-1]}"
group.target_path = chosen
taken.add(chosen)
diff --git a/packages/core/src/repowise/core/generation/concept_tree/naming.py b/packages/core/src/repowise/core/generation/concept_tree/naming.py
index f5a4c8771..731303dfb 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/naming.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/naming.py
@@ -54,9 +54,7 @@ def parse_json_object(content: str) -> dict[str, Any]:
"""
text = (content or "").strip()
if text.startswith("```"):
- text = "\n".join(
- line for line in text.split("\n") if not line.strip().startswith("```")
- )
+ text = "\n".join(line for line in text.split("\n") if not line.strip().startswith("```"))
try:
parsed = json.loads(text)
except (json.JSONDecodeError, ValueError):
@@ -85,7 +83,17 @@ def _humanise(segment: str) -> str:
continue
# Split camelCase so ``pageGenerator`` reads as two words.
parts = re.findall(r"[A-Z]+(?![a-z])|[A-Z][a-z]*|[a-z]+|\d+", word) or [word]
- out.extend(p.capitalize() if p.islower() else p for p in parts)
+ cased = [p.capitalize() if p.islower() else p for p in parts]
+ # A trailing number belongs to the word it was written against: the
+ # ``c4`` directory is the C4 model, and "C 4 Nodes" reads as a
+ # tokeniser leaking into a page title.
+ merged: list[str] = []
+ for part in cased:
+ if part.isdigit() and merged:
+ merged[-1] += part
+ else:
+ merged.append(part)
+ out.extend(merged)
return " ".join(out)
@@ -103,7 +111,12 @@ def deterministic_title(group: ConceptGroup, layer_label: str = "") -> str:
tail = [_humanise(s) for s in meaningful[-2:]]
title = " ".join(tail).strip()
if layer_label and len(title.split()) < MIN_TITLE_WORDS:
- title = f"{layer_label} {title}".strip()
+ # Drop what the label already says. A ``docs`` directory in a "Docs &
+ # Tooling" layer was titled "Docs Tooling Docs", which reads as a
+ # generation bug rather than a name.
+ label_words = {w.lower() for w in layer_label.split()}
+ kept = [w for w in title.split() if w.lower() not in label_words]
+ title = f"{layer_label} {' '.join(kept)}".strip()
if not title:
# The repository root has no path segments to name it by. It is a real
# place and a common one — top-level entry points live there — so it
@@ -117,6 +130,42 @@ def deterministic_title(group: ConceptGroup, layer_label: str = "") -> str:
return title
+def disambiguate_titles(titled: list[tuple[str, str]]) -> list[str]:
+ """Make a set of ``(title, target_path)`` pairs' titles unique.
+
+ Path-derived names collide easily: two packages that each hold a ``ui``
+ directory produce the same last-two-segments name. The pages themselves
+ stay distinct because identity is structural, but a reader sees two
+ identical rows in the tree and cannot tell which is which, and anything
+ that keys on a title (interlinking, an overview listing) has to guess.
+
+ A colliding title takes on the next path segment above the ones it
+ already used, and falls back to its full target path if even that
+ repeats. Input order does not matter: the pairs are resolved in path
+ order, so the same set always produces the same names.
+ """
+ order = sorted(range(len(titled)), key=lambda i: titled[i][1])
+ out = list(titled)
+ used: set[str] = set()
+ for i in order:
+ title, target = titled[i]
+ if title not in used:
+ used.add(title)
+ out[i] = (title, target)
+ continue
+ segments = [s for s in target.split("/") if s]
+ candidate = title
+ for extra in reversed(segments[:-2] or segments[:-1]):
+ candidate = f"{_humanise(extra)} {title}"
+ if candidate not in used:
+ break
+ if candidate in used:
+ candidate = f"{title} ({target})"
+ used.add(candidate)
+ out[i] = (candidate, target)
+ return [t for t, _ in out]
+
+
def deterministic_scope(group: ConceptGroup) -> str:
"""A scope line stating coverage and its boundary, with no model."""
# The root directory is the empty string, which is a real directory and
@@ -210,9 +259,7 @@ def _sample_names(members: list[str], limit: int) -> list[str]:
"""
by_dir: dict[str, list[str]] = {}
for member in sorted(members):
- by_dir.setdefault(member.rsplit("/", 1)[0], []).append(
- member.rsplit("/", 1)[-1]
- )
+ by_dir.setdefault(member.rsplit("/", 1)[0], []).append(member.rsplit("/", 1)[-1])
out: list[str] = []
order = sorted(by_dir)
depth = 0
@@ -406,8 +453,9 @@ def decode_response(
)
)
- named.sort(key=lambda n: (section_rank.get(n.section, len(sections)), n.order,
- n.group.target_path))
+ named.sort(
+ key=lambda n: (section_rank.get(n.section, len(sections)), n.order, n.group.target_path)
+ )
return named, sorted(set(invented))
diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.py
index f69a12959..7fe968368 100644
--- a/packages/core/src/repowise/core/generation/context/assembler.py
+++ b/packages/core/src/repowise/core/generation/context/assembler.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from pathlib import PurePosixPath
from typing import Any
import structlog
@@ -278,7 +279,7 @@ class ContextAssembler:
def assemble_module_page(
self,
- module_path: str,
+ title: str,
language: str,
file_contexts: list[FilePageContext],
graph: Any, # nx.DiGraph
@@ -369,8 +370,19 @@ class ContextAssembler:
key_classes.append(h)
key_classes = key_classes[:10] # cap total
+ # Where the page's files actually live: the directories that hold one
+ # directly, not every ancestor of every file, so the list reads as the
+ # set of places to go and look. Files at the repository root have
+ # ``"."`` for a parent and contribute no entry; a page that is only
+ # root files therefore has none, and the template says so rather than
+ # printing a dot.
+ directories = sorted(
+ {parent for f in files if (parent := str(PurePosixPath(f).parent)) != "."}
+ )
+
return ModulePageContext(
- module_path=module_path,
+ title=title,
+ directories=directories,
language=language,
total_symbols=total_symbols,
public_symbols=public_symbols,
diff --git a/packages/core/src/repowise/core/generation/context/contexts.py b/packages/core/src/repowise/core/generation/context/contexts.py
index d68433a4c..edfb9d724 100644
--- a/packages/core/src/repowise/core/generation/context/contexts.py
+++ b/packages/core/src/repowise/core/generation/context/contexts.py
@@ -81,7 +81,12 @@ class SymbolSpotlightContext:
@dataclass
class ModulePageContext:
- module_path: str
+ # The page's display title. A concept group spans several directories and
+ # is named for what it does, so this is prose ("Ingestion Pipeline") and
+ # not a path. Same split as LayerPageContext's layer_name/layer_id: what
+ # the reader sees is separate from what the page is keyed by, and only the
+ # key is stable across regenerations.
+ title: str
language: str
total_symbols: int
public_symbols: int
@@ -90,6 +95,11 @@ class ModulePageContext:
dependents: list[str]
pagerank_mean: float
files: list[str]
+ # The directories the page covers, shallowest first. The title says what
+ # the page is about; this says where it lives, which is what a reader
+ # needs to go and look. Derived from the members rather than passed in, so
+ # it cannot disagree with them.
+ directories: list[str] = field(default_factory=list)
# Graph intelligence enrichment
file_summaries: dict[str, str] = field(default_factory=dict)
community_label: str = ""
diff --git a/packages/core/src/repowise/core/generation/models.py b/packages/core/src/repowise/core/generation/models.py
index c957a3982..4850a283d 100644
--- a/packages/core/src/repowise/core/generation/models.py
+++ b/packages/core/src/repowise/core/generation/models.py
@@ -98,7 +98,6 @@ class GenerationConfig:
coverage_pct: float = 0.20
file_page_share: float = 0.50
symbol_spotlight_share: float = 0.15
- module_page_share: float = 0.10
api_contract_share: float = 0.08
infra_page_share: float = 0.05
scc_share: float = 0.04
@@ -113,17 +112,6 @@ class GenerationConfig:
file_page_min_symbols: int = 1
skip_trivial_files: bool = True
dedupe_near_clones: bool = True
- # Module-page grouping source. "curated" (default) groups by the wiki
- # modules the KG curation pass derives (stable path ids, human names,
- # right-sized groups) and silently falls back to "community" when no
- # curated modules are available (curation off, degraded, or no KG
- # artifact), so it is always safe. "community" groups by raw graph
- # communities (the pre-curation behavior, kept as the escape hatch),
- # "top_dir" by top-level directory.
- # min_module_size is the floor below which a group doesn't get its own
- # page (its files still appear under file_page).
- module_grouping: Literal["community", "top_dir", "curated"] = "curated"
- min_module_size: int = 3
# Phase 3: emit the curated Onboarding collection at level 8. Each
# subkind defines its own gate; slots whose gates fail are silently
# skipped (no UI nav entry either).
diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.py
index d202d5001..c193e3662 100644
--- a/packages/core/src/repowise/core/generation/page_generator/levels.py
+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py
@@ -278,6 +278,15 @@ def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
for mg in run.sel_module_groups:
fcs = [run.file_page_contexts[fp] for fp in mg.file_paths if fp in run.file_page_contexts]
if not fcs:
+ # A concept group whose files all failed to build a context. The
+ # partition is total, so this is a hole in the tree rather than a
+ # page not worth writing, and it should be visible when it
+ # happens instead of leaving the reader to notice the gap.
+ log.warning(
+ "module_page.skipped_no_file_contexts",
+ target_path=mg.key,
+ members=len(mg.file_paths),
+ )
continue
page_id = compute_page_id("module_page", mg.key)
if not run._emit(page_id):
@@ -300,6 +309,8 @@ def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
community_label=mg.label,
community_cohesion=mg.cohesion,
target_path=mg.key,
+ structural_key=mg.structural_key,
+ members=list(mg.file_paths),
),
)
)
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index dfff264c3..78657045d 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -256,8 +256,9 @@ class _GenerationRun:
kg_file_scores=kg_scores or None,
# Curated wiki modules: prefer the in-memory pipeline
# result (fresh); the artifact file is absent on first init
- # and one run stale on update. Inert unless
- # module_grouping == "curated".
+ # and one run stale on update. Read only for the file -> layer
+ # map that steers concept grouping, so absence costs taste and
+ # never coverage.
kg_modules=self.kg_modules or self.kg_ctx.get_modules() or None,
# Per-file question demand from session transcripts: tilts the
# file_page budget toward modules agents ask about. Empty (no
diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.py
index 0c76e6091..6d2b1b1a1 100644
--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py
+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py
@@ -96,7 +96,7 @@ class PerTypeGenerationMixin:
async def generate_module_page(
self,
- module_path: str,
+ title: str,
language: str,
file_contexts: list[FilePageContext],
graph: Any,
@@ -108,9 +108,11 @@ class PerTypeGenerationMixin:
community_label: str | None = None,
community_cohesion: float | None = None,
target_path: str | None = None,
+ structural_key: str = "",
+ members: list[str] | None = None,
) -> GeneratedPage:
ctx = self._assembler.assemble_module_page(
- module_path,
+ title,
language,
file_contexts,
graph,
@@ -140,17 +142,32 @@ class PerTypeGenerationMixin:
"most_active_file": most_active.get("file_path", ""),
"most_active_commits_90d": most_active.get("commit_count_90d", 0),
}
- page_target = target_path or module_path
+ # The page id is built from this, so it has to be a path, and it has
+ # to be the same path the grouper chose: it computes one shared
+ # directory per group and guarantees the result is unique across
+ # groups. Picking a different one here (the first directory, say)
+ # would mint an id the rest of the run does not agree with, so a
+ # caller that supplies no target gets the group's own anchor or, for
+ # a group with no directory at all, the root name the grouper uses.
+ page_target = target_path or (min(ctx.directories) if ctx.directories else "root")
# The files this page covers. Its own identity and its place in the
- # tree both derive from them: a module's target_path can be a
- # clustering ordinal rather than a directory, in which case the member
- # list is the only thing that says where the page belongs.
- members = sorted(fc.file_path for fc in file_contexts)
+ # tree both derive from them: the page groups several directories, so
+ # its target_path is one directory the group touches rather than a
+ # container for all of it, and the member list is the only thing that
+ # says where the page belongs.
+ #
+ # Taken from the group rather than from ``file_contexts``, and the
+ # difference is load-bearing. A file context only exists for a file the
+ # run built one for, so deriving members here would silently narrow the
+ # page to that subset while ``structural_key`` still hashed the whole
+ # group. Placement resolves file ownership from this list, so the
+ # dropped files would be parented somewhere else entirely, and the two
+ # records of what the page covers would disagree.
+ covered = sorted(members) if members else sorted(fc.file_path for fc in file_contexts)
if self._config.deterministic:
- page = self._deterministic_module_page(
- ctx, page_target, f"Module: {module_path}", module_git_summary
- )
- page.metadata["file_paths"] = members
+ page = self._deterministic_module_page(ctx, page_target, title, module_git_summary)
+ page.metadata["file_paths"] = covered
+ page.structural_key = structural_key or page.structural_key
return page
user_prompt = self._render("module_page.j2", ctx=ctx, module_git_summary=module_git_summary)
response = await self._call_provider(
@@ -159,12 +176,17 @@ class PerTypeGenerationMixin:
page = self._build_generated_page(
"module_page",
page_target,
- f"Module: {module_path}",
+ title,
response,
compute_source_hash(user_prompt),
GENERATION_LEVELS["module_page"],
)
- page.metadata["file_paths"] = members
+ page.metadata["file_paths"] = covered
+ # Set by the producer, preserved by ``_stamp_structural_keys``. The
+ # grouper hashed exactly this member list when it decided what the
+ # group was, so recomputing it downstream would give two places that
+ # must agree about page identity — the arrangement D2 rules out.
+ page.structural_key = structural_key or page.structural_key
return page
async def generate_scc_page(
diff --git a/packages/core/src/repowise/core/generation/related_pages.py b/packages/core/src/repowise/core/generation/related_pages.py
index 209544637..697c0e365 100644
--- a/packages/core/src/repowise/core/generation/related_pages.py
+++ b/packages/core/src/repowise/core/generation/related_pages.py
@@ -61,7 +61,8 @@ def _module_siblings(
for group in module_groups or []:
paths = list(getattr(group, "file_paths", ()) or ())
for path in paths:
- # First group wins — curated groups precede fallbacks upstream.
+ # First group wins. The concept partition is disjoint, so this
+ # only matters if a caller ever passes overlapping groups.
siblings.setdefault(path, [p for p in paths if p != path])
return siblings
diff --git a/packages/core/src/repowise/core/generation/selection/__init__.py b/packages/core/src/repowise/core/generation/selection/__init__.py
index cc3abe266..ee1f44ce2 100644
--- a/packages/core/src/repowise/core/generation/selection/__init__.py
+++ b/packages/core/src/repowise/core/generation/selection/__init__.py
@@ -23,7 +23,6 @@ from .scoring import (
score_api_contract,
score_file,
score_infra,
- score_module,
score_scc,
score_symbol,
)
@@ -46,7 +45,6 @@ __all__ = [
"score_api_contract",
"score_file",
"score_infra",
- "score_module",
"score_scc",
"score_symbol",
"select_pages",
diff --git a/packages/core/src/repowise/core/generation/selection/scoring.py b/packages/core/src/repowise/core/generation/selection/scoring.py
index 9fbc7fd05..91ca58910 100644
--- a/packages/core/src/repowise/core/generation/selection/scoring.py
+++ b/packages/core/src/repowise/core/generation/selection/scoring.py
@@ -15,7 +15,6 @@ from __future__ import annotations
from pathlib import Path
from typing import Any
-
# -- File score --------------------------------------------------------
# Bonuses applied additively on top of normalized PageRank+betweenness.
@@ -123,22 +122,6 @@ def score_symbol(symbol: Any, file_pagerank: float, max_pagerank: float) -> floa
return _normalize(file_pagerank, max_pagerank) * weight
-# -- Module score ------------------------------------------------------
-
-
-def score_module(*, size: int, cohesion: float, min_module_size: int) -> float:
- """Score a community/directory cluster for ``module_page`` candidacy.
-
- Communities below ``min_module_size`` score 0 — they fold into the
- parent directory's module page (or are dropped if there is no
- parent).
- """
- if size < min_module_size:
- return 0.0
- # Cohesion is typically [0, 1]. Multiply by log-ish size factor.
- return (size**0.5) * (0.5 + cohesion)
-
-
# -- SCC score ---------------------------------------------------------
diff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.py
index 80369db07..a854fd56f 100644
--- a/packages/core/src/repowise/core/generation/selection/selector.py
+++ b/packages/core/src/repowise/core/generation/selection/selector.py
@@ -8,7 +8,8 @@ emitted.
from __future__ import annotations
-from dataclasses import dataclass, field
+from collections import Counter
+from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any
@@ -16,6 +17,8 @@ import structlog
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
+from ..concept_tree.grouping import group_files
+from ..concept_tree.naming import deterministic_title, disambiguate_titles
from ..models import scc_page_slug
from ..tour import DEFAULT_MAX_LANDMARKS, tour_landmark_paths
from .budget import (
@@ -29,7 +32,6 @@ from .scoring import (
score_api_contract,
score_file,
score_infra,
- score_module,
score_scc,
score_symbol,
)
@@ -50,10 +52,17 @@ _CODE_LANGUAGES = _LANG_REGISTRY.code_languages()
class ModuleGroup:
"""One ``module_page`` worth of files.
- ``key`` is the stable identifier persisted as ``target_path``:
- the module's real directory path when grouping by curated KG modules
- (so path-prefix child lookups work), ``community-<id>`` when grouping
- by community, the top-level directory when falling back to ``top_dir``.
+ ``key`` is the stable identifier persisted as ``target_path``, and it is
+ always a real directory: the shallowest directory the group's members
+ share. The bench gold set matches pages by exact equality against the
+ surfaced ``target_path``, so a group that merged several sibling
+ directories still has to name one of them.
+
+ ``structural_key`` is where the abstract identity lives — a hash of the
+ sorted member list, carrying the ``concept-`` prefix minted by the
+ grouper. It is set by whatever produced the group rather than recomputed
+ downstream, because two places agreeing about page identity is the
+ arrangement D2 exists to prevent.
"""
key: str
@@ -62,6 +71,7 @@ class ModuleGroup:
file_paths: tuple[str, ...]
label: str | None = None
cohesion: float | None = None
+ structural_key: str = ""
@dataclass
@@ -127,9 +137,10 @@ class SelectionInputs:
config: Any # GenerationConfig — duck-typed to avoid the import cycle
kg_file_scores: dict[str, float] | None = None
# Curated wiki modules from the KG artifact (``modules`` top-level key),
- # passed through — never re-derived here. Only read when
- # ``config.module_grouping == "curated"``; ``None``/empty falls back to
- # community grouping (the fallback-matrix "degraded" row).
+ # passed through — never re-derived here. Read for one thing only: the
+ # file -> layer map that steers which adjacent directories merge into a
+ # concept group. Absent or partial degrades the grouping's taste, never
+ # its coverage, because the partition itself needs no KG input.
kg_modules: list[dict] | None = None
# Per-file question demand mined from session transcripts
# (``core.sessions.miners.demand.aggregate_file_demand``): repo-relative
@@ -182,24 +193,6 @@ def _passes_tail_floor(path: str, tail_dirs: tuple[str, ...] | None) -> bool:
return True
-def _is_test_group(files: list[Any]) -> bool:
- """Whether a module group is mostly test files.
-
- Test directories do not deserve a module page. The same floor already
- applies to the deterministic coverage tail, where excluding test files
- raised retrieval rather than lowering it: they add pages that match on
- vocabulary without carrying an answer. Tests keep their own file pages,
- which is what an agent actually needs when it goes looking for one.
-
- Majority rather than any, so a production module with a colocated test
- beside it (the Go and Jest convention) keeps its page.
- """
- if not files:
- return False
- tests = sum(1 for p in files if getattr(p.file_info, "is_test", False))
- return tests * 2 > len(files)
-
-
def _select_deterministic_tail(
files: list[tuple[float, str]],
selected_files: list[str],
@@ -287,136 +280,94 @@ def _build_symbol_candidates(
return deduped
-def _build_curated_module_groups(
- inputs: SelectionInputs, min_size: int
-) -> list[tuple[float, ModuleGroup]] | None:
- """Scored groups from curated KG modules, or ``None`` when unavailable.
+def _layer_map_from_kg(
+ inputs: SelectionInputs,
+) -> tuple[dict[str, str], dict[str, str]]:
+ """File -> curated layer id, and layer id -> display name.
- ``key`` = the module's real directory path (its ``target_path``, so the
- MCP child lookup ``target_path LIKE 'dir/%'`` works), ``display``/``label``
- = the curated human name, ``cohesion`` = None (the community block in the
- template is conditional and has no meaning for directory groups). Ranked
- by Σ PageRank of members — better than ``score_module``'s cohesion term,
- which is meaningless here. Returns ``None`` only when no curated modules
- were passed in (→ community fallback); an artifact whose modules all fall
- below the floor yields an empty list, not a vocabulary mix.
+ The layer signal only steers which adjacent runs of directories merge; it
+ never forces a split, so an absent or partial map degrades the grouping's
+ taste rather than its correctness. That is why this reads whatever the KG
+ artifact happens to carry instead of requiring it.
"""
- if not inputs.kg_modules:
- return None
-
- code_by_path = {p.file_info.path: p for p in inputs.parsed_files if _is_code_file(p)}
- scored: list[tuple[float, ModuleGroup]] = []
- seen_keys: set[str] = set()
- for module in inputs.kg_modules:
- if module.get("wholeLayer"):
- # 1:1 with a layer page (single-module layers, flat libs) — a
- # module doc would re-document the layer. Skip the page; the
- # module stays in the KG artifact for canvas/coverage.
+ layer_of_file: dict[str, str] = {}
+ for module in inputs.kg_modules or []:
+ layer_id = module.get("layerId")
+ if not isinstance(layer_id, str) or not layer_id:
continue
- member_paths = sorted(
- path
- for nid in module.get("nodeIds", [])
- if isinstance(nid, str) and (path := nid.removeprefix("file:")) in code_by_path
- )
- if len(member_paths) < min_size:
- continue
- key = module.get("path") or (module.get("id") or "").removeprefix("module:")
- if not key or key in seen_keys:
- # Rare cross-layer dir collision: the slug id is still unique.
- key = (module.get("id") or "").removeprefix("module:")
- if not key or key in seen_keys:
- continue
- seen_keys.add(key)
- name = module.get("name") or key
- language = module.get("language") or code_by_path[member_paths[0]].file_info.language
- score = sum(inputs.pagerank.get(p, 0.0) for p in member_paths)
- scored.append(
- (
- score,
- ModuleGroup(
- key=key,
- display=name,
- language=language,
- file_paths=tuple(member_paths),
- label=name,
- cohesion=None,
- ),
- )
- )
- scored.sort(key=lambda x: (-x[0], x[1].key))
- return scored
+ for nid in module.get("nodeIds", []):
+ if isinstance(nid, str) and nid.startswith("file:"):
+ layer_of_file[nid[5:]] = layer_id
+ labels = {
+ lid: lid.removeprefix("layer:").replace("-", " ").replace("_", " ").title()
+ for lid in set(layer_of_file.values())
+ }
+ return layer_of_file, labels
def _build_module_groups(inputs: SelectionInputs) -> list[tuple[float, ModuleGroup]]:
- """Return scored module groups — curated, community-based, or top-dir."""
- cfg = inputs.config
- min_size = max(1, getattr(cfg, "min_module_size", 3))
- grouping = getattr(cfg, "module_grouping", "community")
-
- if grouping == "curated":
- curated = _build_curated_module_groups(inputs, min_size)
- if curated is not None:
- return curated
- grouping = "community" # no curated artifact → today's path, unchanged
-
- use_communities = grouping == "community"
-
- # Bucket files into groups.
- groups: dict[str, list[Any]] = {}
- group_lang: dict[str, str] = {}
- group_label: dict[str, str | None] = {}
- group_cohesion: dict[str, float | None] = {}
- group_display: dict[str, str] = {}
-
- if use_communities and inputs.community_info:
- for p in inputs.parsed_files:
- if not _is_code_file(p):
- continue
- cid = inputs.community.get(p.file_info.path)
- if cid is None:
- continue
- key = f"community-{cid}"
- groups.setdefault(key, []).append(p)
- group_lang.setdefault(key, p.file_info.language)
- if key not in group_display:
- ci = inputs.community_info.get(cid)
- label = getattr(ci, "label", "") or f"cluster_{cid}"
- group_display[key] = label
- group_label[key] = label
- group_cohesion[key] = float(getattr(ci, "cohesion", 0.0) or 0.0)
- else:
- for p in inputs.parsed_files:
- if not _is_code_file(p):
- continue
- parts = Path(p.file_info.path).parts
- key = parts[0] if len(parts) > 1 else "root"
- groups.setdefault(key, []).append(p)
- group_lang.setdefault(key, p.file_info.language)
- group_display.setdefault(key, key)
- group_label.setdefault(key, None)
- group_cohesion.setdefault(key, None)
+ """Return the concept groups, scored, one per ``module_page``.
+
+ This is the concept tree, not one page per directory. The partition comes
+ from :func:`group_files`, which bins the production files into bounded,
+ path-local subtrees with no model in the loop, so it is total (every
+ non-test production file lands in exactly one group) and deterministic
+ (the same file set always produces the same groups, and therefore the same
+ page identities).
+
+ Totality is the property that makes this a replacement rather than an
+ alternative. The per-directory grouping this supersedes covered a subset
+ of the tree under a worse grouping and gave eleven of its pages to test
+ directories, which D8 measured as pure dilution. There is no coexistence
+ mode: D1 forbids one page type existing in two forms, and a wiki holding
+ both populations would be exactly that.
+
+ Scored by summed PageRank so the most central subsystem sorts first. The
+ score no longer rations anything — see :func:`select_pages` — but page
+ order is still what the reader sees first.
+ """
+ files = [
+ p.file_info.path
+ for p in inputs.parsed_files
+ if _is_code_file(p) and not getattr(p.file_info, "is_test", False)
+ ]
+ if not files:
+ return []
+ layer_of_file, layer_labels = _layer_map_from_kg(inputs)
+ groups = group_files(files, layer_of_file=layer_of_file)
+
+ lang_of = {p.file_info.path: p.file_info.language for p in inputs.parsed_files}
+ # Names are decided over the whole set, not per group: two packages that
+ # each hold a ``ui`` directory derive the same name from their paths, and
+ # two identical rows in the tree are indistinguishable to a reader even
+ # though the pages behind them are not.
+ titles = disambiguate_titles(
+ [
+ (deterministic_title(g, layer_labels.get(g.dominant_layer, "")), g.target_path)
+ for g in groups
+ ]
+ )
scored: list[tuple[float, ModuleGroup]] = []
- for key, files in groups.items():
- if _is_test_group(files):
- continue
- s = score_module(
- size=len(files),
- cohesion=group_cohesion.get(key) or 0.0,
- min_module_size=min_size,
- )
- if s <= 0.0:
- continue
+ for group, title in zip(groups, titles, strict=True):
+ langs = Counter(lang_of.get(m, "") for m in group.members)
+ langs.pop("", None)
+ score = sum(inputs.pagerank.get(m, 0.0) for m in group.members)
scored.append(
(
- s,
+ score,
ModuleGroup(
- key=key,
- display=group_display.get(key, key),
- language=group_lang.get(key, "unknown"),
- file_paths=tuple(sorted(p.file_info.path for p in files)),
- label=group_label.get(key),
- cohesion=group_cohesion.get(key),
+ # Taken as given. The grouper guarantees it is non-empty
+ # and distinct across groups, and naming the root here
+ # instead would sit outside the uniqueness check that
+ # makes the guarantee.
+ key=group.target_path,
+ display=title,
+ language=langs.most_common(1)[0][0] if langs else "unknown",
+ file_paths=tuple(group.members),
+ label=None,
+ cohesion=None,
+ structural_key=group.structural_key,
),
)
)
@@ -469,7 +420,10 @@ def _shares_from_config(cfg: Any) -> dict[str, float]:
return {
"file_page": getattr(cfg, "file_page_share", 0.50),
"symbol_spotlight": getattr(cfg, "symbol_spotlight_share", 0.15),
- "module_page": getattr(cfg, "module_page_share", 0.10),
+ # Zero, and not a knob. The concept pages are taken whole, so a share
+ # here would reserve budget for a bucket that does not spend it and
+ # quietly shrink the file-page coverage the user actually asked for.
+ "module_page": 0.0,
"api_contract": getattr(cfg, "api_contract_share", 0.08),
"infra_page": getattr(cfg, "infra_page_share", 0.05),
"scc_page": getattr(cfg, "scc_share", 0.04),
@@ -691,6 +645,11 @@ def select_pages(inputs: SelectionInputs) -> Selection:
shares=_shares_from_config(cfg),
n_files=len(inputs.parsed_files),
)
+ # The module bucket is not rationed (see the Selection below), so the
+ # allocation has to say so. Leaving the budgeted number here would make
+ # the cost estimator quote a figure the run does not honour, which is the
+ # one thing the "allow-set honored verbatim" contract exists to prevent.
+ allocation = replace(allocation, module_page=len(modules))
# File pages, tilted toward the modules agents ask about most (demand-free
# top-K when there is no session data, so fresh installs are unchanged).
@@ -720,7 +679,14 @@ def select_pages(inputs: SelectionInputs) -> Selection:
file_page_paths=selected_files,
deterministic_tail_paths=deterministic_tail,
symbol_spotlights=[t for _, t in symbols[: allocation.symbol_spotlight]],
- module_groups=[m for _, m in modules[: allocation.module_page]],
+ # Never truncated, unlike every other bucket. The concept groups are a
+ # total partition of the production files, so rationing them does not
+ # produce a smaller wiki — it produces one with files that belong to no
+ # page at all, and the tree stops being a map of the repository. The
+ # cost of not rationing is bounded by construction rather than by a
+ # budget: the grouper's size ladder holds the page count near 50-80 for
+ # a repository this size and scales sublinearly above it.
+ module_groups=[m for _, m in modules],
api_contract_paths=[p for _, p in apis[: allocation.api_contract]],
infra_paths=[p for _, p in infras[: allocation.infra_page]],
scc_groups=[g for _, g in sccs[: allocation.scc_page]],
diff --git a/packages/core/src/repowise/core/generation/templates/deterministic/module_page.j2 b/packages/core/src/repowise/core/generation/templates/deterministic/module_page.j2
index 09ae38a7e..e8312322d 100644
--- a/packages/core/src/repowise/core/generation/templates/deterministic/module_page.j2
+++ b/packages/core/src/repowise/core/generation/templates/deterministic/module_page.j2
@@ -5,7 +5,9 @@
Overview is synthesised from the module's own signals (centrality, entry
points, cluster) and the structural sections carry the rest.
-#}
-# Module: {{ ctx.module_path }}
+# {{ ctx.title }}
+
+{% if ctx.directories %}{% for d in ctx.directories[:6] %}`{{ d }}`{% if not loop.last %} · {% endif %}{% endfor %}{% if ctx.directories | length > 6 %} · and {{ ctx.directories | length - 6 }} more{% endif %}{% else %}Repository root{% endif %}
**Language:** {{ ctx.language }} | **Files:** {{ ctx.files | length }} | **Public symbols:** {{ ctx.public_symbols }} / {{ ctx.total_symbols }}{% if ctx.community_label %} | **Cluster:** {{ ctx.community_label }}{% if ctx.community_cohesion > 0 %} (cohesion {{ "%.2f"|format(ctx.community_cohesion) }}){% endif %}{% endif %}
@@ -16,7 +18,7 @@
{%- if ctx.pagerank_mean is defined and ctx.pagerank_mean >= 0.005 %}{% set _ = _role.append("sits centrally in the import graph (mean PageRank " ~ "%.4f"|format(ctx.pagerank_mean) ~ ")") %}{% endif -%}
{%- if ctx.dependents %}{% set _ = _role.append("is imported by " ~ ctx.dependents | length ~ " other module" ~ ("s" if ctx.dependents | length != 1 else "")) %}{% endif -%}
{%- if ctx.dependencies %}{% set _ = _role.append("draws on " ~ ctx.dependencies | length ~ " internal module" ~ ("s" if ctx.dependencies | length != 1 else "")) %}{% endif -%}
-`{{ ctx.module_path }}` groups {{ ctx.files | length }} {{ ctx.language }} file{{ "s" if ctx.files | length != 1 else "" }} exposing {{ ctx.public_symbols }} public symbol{{ "s" if ctx.public_symbols != 1 else "" }}.{% if _role %} It {{ _role | join("; ") }}.{% endif %}
+{{ ctx.title }} covers {{ ctx.files | length }} {{ ctx.language }} file{{ "s" if ctx.files | length != 1 else "" }}{% if ctx.directories %} across {{ ctx.directories | length }} director{{ "ies" if ctx.directories | length != 1 else "y" }}{% endif %}, exposing {{ ctx.public_symbols }} public symbol{{ "s" if ctx.public_symbols != 1 else "" }}.{% if _role %} It {{ _role | join("; ") }}.{% endif %}
{% if ctx.entry_points %}
## Entry Points
diff --git a/packages/core/src/repowise/core/generation/templates/module_page.j2 b/packages/core/src/repowise/core/generation/templates/module_page.j2
index b05f554fd..ecbcbe5da 100644
--- a/packages/core/src/repowise/core/generation/templates/module_page.j2
+++ b/packages/core/src/repowise/core/generation/templates/module_page.j2
@@ -1,8 +1,8 @@
-You are documenting the module **{{ ctx.module_path }}** (language: {{ ctx.language }}).
+You are documenting **{{ ctx.title }}** (language: {{ ctx.language }}), a group of files that belong together, not a single directory. Write about what the group does as a whole.
## Module Overview
-**Module:** `{{ ctx.module_path }}` | **Language:** {{ ctx.language }} | **Files:** {{ ctx.files | length }} | **Public symbols:** {{ ctx.public_symbols }} / {{ ctx.total_symbols }}{% if ctx.community_label %} | **Community:** {{ ctx.community_label }}{% if ctx.community_cohesion > 0 %} (cohesion {{ "%.2f"|format(ctx.community_cohesion) }}){% endif %}{% endif %}
+**Covers:** {% if ctx.directories %}{% for d in ctx.directories[:6] %}`{{ d }}`{% if not loop.last %}, {% endif %}{% endfor %}{% if ctx.directories | length > 6 %} and {{ ctx.directories | length - 6 }} more{% endif %}{% else %}the repository root{% endif %} | **Language:** {{ ctx.language }} | **Files:** {{ ctx.files | length }} | **Public symbols:** {{ ctx.public_symbols }} / {{ ctx.total_symbols }}{% if ctx.community_label %} | **Community:** {{ ctx.community_label }}{% if ctx.community_cohesion > 0 %} (cohesion {{ "%.2f"|format(ctx.community_cohesion) }}){% endif %}{% endif %}
{# The role-leading sentence is the highest-leverage signal a module page
carries. Surface the module's position in the system so the LLM can frame
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index 725b0d3dd..baa532d55 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -21,7 +21,7 @@ import threading
from collections.abc import Iterator
from dataclasses import dataclass, field
from datetime import datetime
-from pathlib import Path
+from pathlib import Path, PurePosixPath
import pathspec
import structlog
@@ -197,6 +197,28 @@ _ENTRY_POINT_NAME_SUFFIXES: tuple[str, ...] = tuple(
sorted(p[1:] for p in _LANG_REGISTRY.entry_point_names() if p.startswith("*"))
)
+# Test-file conventions, from the same registry each language declares them
+# on. ``spec_`` is carried over from the rule these replaced: the registry
+# has the ``_spec`` suffix and the ``spec_helper`` stem but no prefix, and
+# dropping it would unclassify RSpec-style ``spec_foo.rb``.
+_TEST_STEM_PREFIXES: tuple[str, ...] = tuple(
+ sorted({*_LANG_REGISTRY.test_stem_prefixes(), "spec_"})
+)
+_TEST_STEM_SUFFIXES: tuple[str, ...] = _LANG_REGISTRY.test_stem_suffixes()
+_TEST_INFIXES: tuple[str, ...] = _LANG_REGISTRY.test_infixes()
+_TEST_FIXTURE_STEMS: frozenset[str] = _LANG_REGISTRY.test_fixture_stems()
+# Directory names that mark everything beneath them as test material.
+# ``__tests__`` is the JavaScript convention and is the reason this set is
+# not simply the registry's per-language tokens.
+_TEST_DIR_SEGMENTS: frozenset[str] = frozenset(
+ {"test", "tests", "__tests__", "__test__"}
+ | {
+ token
+ for tokens in _LANG_REGISTRY.test_dir_tokens_by_language().values()
+ for token in tokens
+ }
+)
+
# Default file-size limit
_DEFAULT_MAX_FILE_SIZE_BYTES: int = 500 * 1024 # 500 KB
@@ -650,13 +672,29 @@ def _is_generated(abs_path: Path) -> bool:
def _is_test_file(rel_path: str, filename: str) -> bool:
- stem = Path(filename).stem.lower()
- if stem.startswith("test_") or stem.endswith("_test"):
- return True
- if stem.startswith("spec_") or stem.endswith("_spec"):
+ """Whether *rel_path* is test material, by directory or by filename.
+
+ Conventions come from the language registry, which is where each
+ language already declares them, rather than from a second hard-coded
+ list. The two rules this replaced knew only ``test_x`` / ``x_test`` and a
+ slash-delimited ``/tests/`` substring, so both halves of the JavaScript
+ convention (``__tests__/`` directories and ``x.test.ts`` filenames) and
+ every root-level suite (``tests/conftest.py``) read as production code to
+ everything downstream of this flag.
+ """
+ name = filename.lower()
+ stem = PurePosixPath(name).stem
+ if (
+ stem in _TEST_FIXTURE_STEMS
+ or stem.startswith(_TEST_STEM_PREFIXES)
+ or stem.endswith(_TEST_STEM_SUFFIXES)
+ or any(infix in name for infix in _TEST_INFIXES)
+ ):
return True
- path_lower = rel_path.lower()
- return "/test/" in path_lower or "/tests/" in path_lower or "/spec/" in path_lower
+ # Directory segments, including the first: a repository-root ``tests/``
+ # is the usual layout, not the exception.
+ segments = rel_path.lower().replace("\\", "/").split("/")[:-1]
+ return any(seg in _TEST_DIR_SEGMENTS for seg in segments)
def _is_config_file(language: LanguageTag) -> bool:
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
index 3424b8c18..ff9e84556 100644
--- a/packages/core/src/repowise/core/pipeline/orchestrator.py
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -298,9 +298,7 @@ async def run_pipeline(
try:
graph_builder, git_meta_map = await resume_controller.rehydrate_index(repo_path)
if progress:
- progress.on_message(
- "info", " ↳ Resuming — reusing persisted graph + git index"
- )
+ progress.on_message("info", " ↳ Resuming — reusing persisted graph + git index")
(
parsed_files,
file_infos,
@@ -601,8 +599,8 @@ async def run_pipeline(
# ---- Phase 3: Generation (optional) ------------------------------------
generated_pages: list[Any] | None = None
# Structural page types this run was authoritative for (see
- # PipelineResult.authoritative_page_types). Stays empty unless curated
- # generation engaged below.
+ # PipelineResult.authoritative_page_types). Stays empty unless generation
+ # runs below, because a run that generated nothing decided nothing.
authoritative_page_types: set[str] = set()
# DETERMINISTIC supplies its own null provider so a caller with no key can
# still get a wiki. Scoped to this phase rather than assigned over
@@ -642,9 +640,7 @@ async def run_pipeline(
if mode.deterministic_docs and not resolved_generation_config.deterministic:
from dataclasses import replace as _dc_replace
- resolved_generation_config = _dc_replace(
- resolved_generation_config, deterministic=True
- )
+ resolved_generation_config = _dc_replace(resolved_generation_config, deterministic=True)
# Phase 2 enrichment: flag framework-defined HTTP surfaces (FastAPI,
# ASP.NET controllers, …) as api_contract so they route through the
@@ -727,9 +723,7 @@ async def run_pipeline(
else None
),
kg_data=(
- knowledge_graph_result.to_dict()
- if knowledge_graph_result is not None
- else None
+ knowledge_graph_result.to_dict() if knowledge_graph_result is not None else None
),
)
except BaseException:
@@ -744,19 +738,33 @@ async def run_pipeline(
# Record which structural page types this run authoritatively decided,
# so the sweep can retire prior rows of a type even when this run
- # legitimately emitted zero pages of it. Mirrors the selector's curated
- # engagement test (_build_curated_module_groups returns None — i.e.
- # falls back to community — only when kg_modules is empty) so the signal
- # is set iff the curated grouping actually engaged. On the degraded
- # community fallback both stay unset, preserving degradation honesty.
- kg_modules_present = bool(
- knowledge_graph_result is not None and knowledge_graph_result.modules
+ # legitimately emitted zero pages of it.
+ #
+ # ``module_page`` needs no KG and no budget: the concept grouping
+ # partitions every production file, so a run that produced any concept
+ # page produced all of them and prior rows should go. The condition
+ # that used to stand here tested whether a curated KG artifact had
+ # engaged a grouping mode that no longer exists, which stranded the
+ # superseded pages on exactly the repositories the KG failed on.
+ #
+ # It is still not unconditional, because "emitted zero" has causes
+ # other than "there is nothing to document": ``file_pages_only``
+ # returns before the concept level runs at all, and a provider error
+ # on that level is logged per page rather than raised. Claiming
+ # authority there would delete every concept page over a 429. So the
+ # claim is made on evidence — at least one page produced — with the
+ # genuinely empty repository as the one exception, since it has no
+ # page to show and its stale rows are exactly what wants sweeping.
+ produced_module_page = any(
+ getattr(page, "page_type", "") == "module_page" for page in (generated_pages or [])
+ )
+ has_code_to_group = any(
+ not getattr(p.file_info, "is_test", False) for p in (parsed_files or [])
)
kg_layers_present = bool(
knowledge_graph_result is not None and knowledge_graph_result.layers
)
- module_grouping = getattr(resolved_generation_config, "module_grouping", "community")
- if module_grouping == "curated" and kg_modules_present:
+ if produced_module_page or not has_code_to_group:
authoritative_page_types.add("module_page")
if kg_layers_present:
authoritative_page_types.add("layer_page")
diff --git a/tests/unit/cli/test_workspace_curated_grouping.py b/tests/unit/cli/test_workspace_kg_artifact_wiring.py
similarity index 82%
rename from tests/unit/cli/test_workspace_curated_grouping.py
rename to tests/unit/cli/test_workspace_kg_artifact_wiring.py
index 26d1538f6..c46b06978 100644
--- a/tests/unit/cli/test_workspace_curated_grouping.py
+++ b/tests/unit/cli/test_workspace_kg_artifact_wiring.py
@@ -1,10 +1,11 @@
-"""Tests for curated-grouping wiring in ``repowise workspace add``.
-
-``module_grouping="curated"`` is the default. Doc generation can only engage
-curated module grouping if (a) the KG artifact (``.repowise/knowledge-graph.json``)
-was saved during indexing and (b) the generator is told the ``repo_path`` so it
-can load that artifact. These tests pin both seams so a regression to community
-grouping is caught.
+"""Tests for knowledge-graph wiring in ``repowise workspace add``.
+
+Doc generation reads the curated KG for the file -> layer map that steers
+which adjacent directories merge into a concept page. It gets there only if
+(a) the artifact (``.repowise/knowledge-graph.json``) was saved during
+indexing and (b) the generator is told the ``repo_path`` so it can load it.
+Neither seam changes coverage if it breaks, which is exactly why a break is
+easy to miss: the wiki still has a page for every file, grouped worse.
"""
from __future__ import annotations
@@ -16,7 +17,7 @@ from repowise.cli.commands.workspace_cmd import _generate_docs_for_added_repo
def test_do_index_save_idiom_writes_artifact(tmp_path):
- """The ``_do_index`` save idiom must persist the curated KG artifact.
+ """The ``_do_index`` save idiom must persist the KG artifact.
``_do_index`` is a nested closure, so we exercise the exact save call it
performs: when the pipeline result carries a ``knowledge_graph_result``,
@@ -35,7 +36,7 @@ def test_do_index_save_idiom_writes_artifact(tmp_path):
def test_generate_docs_for_added_repo_passes_repo_path(tmp_path):
- """``generate_all`` must receive ``repo_path`` so the curated KG loads."""
+ """``generate_all`` must receive ``repo_path`` so the KG artifact loads."""
repo_path = tmp_path
generator = MagicMock()
diff --git a/tests/unit/generation/test_concept_grouping.py b/tests/unit/generation/test_concept_grouping.py
index 1720496e8..6cdfb8e37 100644
--- a/tests/unit/generation/test_concept_grouping.py
+++ b/tests/unit/generation/test_concept_grouping.py
@@ -119,9 +119,7 @@ class TestStability:
def test_input_order_does_not_change_the_grouping(self):
files = _repo(SPEC)
forward = [g.structural_key for g in group_files(files, params=TINY)]
- reverse = [
- g.structural_key for g in group_files(list(reversed(files)), params=TINY)
- ]
+ reverse = [g.structural_key for g in group_files(list(reversed(files)), params=TINY)]
assert forward == reverse
def test_one_added_file_does_not_reshuffle_the_tree(self):
@@ -134,8 +132,7 @@ class TestStability:
files = _repo(SPEC)
before = {g.structural_key for g in group_files(files, params=TINY)}
after = {
- g.structural_key
- for g in group_files([*files, "src/alpha/new_thing.py"], params=TINY)
+ g.structural_key for g in group_files([*files, "src/alpha/new_thing.py"], params=TINY)
}
assert len(before - after) == 1, "more than one group changed identity"
assert len(after - before) == 1
@@ -195,26 +192,47 @@ class TestIdentity:
def test_target_path_stays_a_real_directory(self):
files = _repo(SPEC)
- directories = {f.rsplit("/", 1)[0] if "/" in f else "" for f in files}
+ # The repository root is a real directory whose path is the empty
+ # string, and the empty string cannot be a page id, so it is the one
+ # target that is a name rather than a path.
+ directories = {f.rsplit("/", 1)[0] for f in files if "/" in f} | {"root"}
for group in group_files(files, params=TINY):
- assert group.target_path in directories, (
- f"{group.target_path!r} is not a directory in this repository"
- )
+ assert (
+ group.target_path in directories
+ ), f"{group.target_path!r} is not a directory in this repository"
- def test_a_group_of_root_level_files_is_targeted_at_the_root(self):
+ def test_a_group_of_root_level_files_is_named_for_the_root(self):
"""The repository root is a directory whose path is the empty string.
Treating that as "no directory found" sent the group to the
identity-hash fallback, which then became its page id, its title
- ("D 73789 F 001 Bd") and its scope sentence.
+ ("D 73789 F 001 Bd") and its scope sentence. It cannot keep the empty
+ string either, because the page id is built from the target and an
+ empty one mints ``module_page:``, so it gets a name here where
+ uniqueness is enforced.
"""
files = _repo({"": 3, "src/pkg": 10})
groups = group_files(files, params=GroupingParams(min_files=2, max_files=4))
root = next(g for g in groups if "setup" in " ".join(g.members) or "f00.py" in g.members)
- assert root.target_path == ""
- assert "#" not in root.target_path
+ assert root.target_path == "root"
+ assert all(g.target_path for g in groups)
assert all("#" not in g.target_path for g in groups)
+ def test_a_repository_with_its_own_root_directory_still_gets_unique_targets(self):
+ """A top-level directory literally called ``root`` is not a conflict.
+
+ The root group's name is picked against the directories that exist,
+ so it steps aside rather than colliding — a collision here would mint
+ one page id for two pages and silently drop one on persist.
+ """
+ files = _repo({"": 3, "root": 6, "src/pkg": 6})
+ groups = group_files(files, params=GroupingParams(min_files=2, max_files=8))
+ targets = [g.target_path for g in groups]
+
+ assert len(targets) == len(set(targets)), targets
+ assert "root" in targets
+ assert any(t.startswith("_root") for t in targets), targets
+
def test_structural_key_follows_members_not_target(self):
a = group_files(_repo({"src/one": 4}), params=TINY)[0]
b = group_files(_repo({"src/two": 4}), params=TINY)[0]
@@ -341,7 +359,5 @@ class TestLayers:
files = _repo({"src/x": 4})
layers = {f: ("layer:a" if i % 2 else "layer:b") for i, f in enumerate(files)}
first = group_files(files, layer_of_file=layers, params=TINY)[0]
- second = group_files(
- list(reversed(files)), layer_of_file=layers, params=TINY
- )[0]
+ second = group_files(list(reversed(files)), layer_of_file=layers, params=TINY)[0]
assert first.dominant_layer == second.dominant_layer
diff --git a/tests/unit/generation/test_concept_outline.py b/tests/unit/generation/test_concept_outline.py
index 35ad33e9e..f25dafb29 100644
--- a/tests/unit/generation/test_concept_outline.py
+++ b/tests/unit/generation/test_concept_outline.py
@@ -121,8 +121,7 @@ class TestModelCannotBreakStructure:
groups = group_files(FILES, params=TINY)
_payload, index = build_payload(groups)
named, _ = decode_response(
- {"sections": [{"title": "Core", "groups": [{"a": 1}, ["b"], None]}],
- "names": {}},
+ {"sections": [{"title": "Core", "groups": [{"a": 1}, ["b"], None]}], "names": {}},
index,
)
assert len(named) == len(index)
@@ -142,8 +141,7 @@ class TestModelCannotBreakStructure:
@pytest.mark.asyncio
async def test_junk_types_in_the_response_are_ignored(self):
content = (
- '{"sections": {"not": "a list"},'
- ' "names": {"g01": ["not", "a", "dict"], "g02": 7}}'
+ '{"sections": {"not": "a list"}, "names": {"g01": ["not", "a", "dict"], "g02": 7}}'
)
outline, report = await plan_outline(
_inputs(), provider=FakeProvider(content), params=TINY, repair=False
@@ -235,8 +233,7 @@ class TestValidator:
ConceptSection(
title="All",
pages=[
- ConceptPage(title=f"Page {i}", scope="s", group=g)
- for i, g in enumerate(groups)
+ ConceptPage(title=f"Page {i}", scope="s", group=g) for i, g in enumerate(groups)
],
)
)
@@ -324,8 +321,7 @@ class TestPayload:
from repowise.core.generation.concept_tree.grouping import ConceptGroup
group = ConceptGroup(
- members=[f"src/aaa/a{i}.py" for i in range(6)]
- + [f"src/zzz/z{i}.py" for i in range(6)],
+ members=[f"src/aaa/a{i}.py" for i in range(6)] + [f"src/zzz/z{i}.py" for i in range(6)],
dirs=["src/aaa", "src/zzz"],
target_path="src/aaa",
)
@@ -344,3 +340,43 @@ class TestPayload:
)
payload, _ = build_payload([group])
assert sorted(payload["groups"][0]["subdirs"]) == ["git_indexer", "graph"]
+
+
+def test_layer_prefixed_title_does_not_repeat_the_layer_word():
+ """A ``docs`` group in a "Docs & Tooling" layer is not "Docs Tooling Docs".
+
+ The layer prefix exists to disambiguate a one-word directory name. When
+ the label already contains that word the prefix adds nothing and the
+ repetition reads as a generation bug.
+ """
+ from repowise.core.generation.concept_tree.grouping import ConceptGroup
+ from repowise.core.generation.concept_tree.naming import deterministic_title
+
+ group = ConceptGroup(
+ members=["docs/conf.py", "docs/build.py"],
+ dirs=["docs"],
+ target_path="docs",
+ )
+ title = deterministic_title(group, "Docs Tooling")
+
+ assert title == "Docs Tooling"
+ assert title.lower().split().count("docs") == 1
+
+
+def test_a_trailing_number_stays_with_its_word():
+ """``c4`` is the C4 model, not a C and a 4.
+
+ The camel-case splitter treats a digit run as its own token, which is
+ right for ``v2Api`` and wrong for every directory whose name ends in a
+ number. Titles are the one place that leak is visible.
+ """
+ from repowise.core.generation.concept_tree.grouping import ConceptGroup
+ from repowise.core.generation.concept_tree.naming import deterministic_title
+
+ group = ConceptGroup(
+ members=["src/ui/c4/nodes/a.ts", "src/ui/c4/nodes/b.ts"],
+ dirs=["src/ui/c4/nodes"],
+ target_path="src/ui/c4/nodes",
+ )
+
+ assert deterministic_title(group) == "C4 Nodes"
diff --git a/tests/unit/generation/test_deterministic_templates.py b/tests/unit/generation/test_deterministic_templates.py
index 86d1cdbf5..1591bfad5 100644
--- a/tests/unit/generation/test_deterministic_templates.py
+++ b/tests/unit/generation/test_deterministic_templates.py
@@ -374,3 +374,67 @@ def test_as_markdown_is_idempotent():
):
once = as_markdown(src)
assert as_markdown(once) == once
+
+
+# ---------------------------------------------------------------------------
+# The concept page's own header
+# ---------------------------------------------------------------------------
+
+
+def _module_ctx(assembler, files):
+ from repowise.core.generation.context_assembler import FilePageContext
+
+ contexts = [
+ FilePageContext(
+ file_path=f,
+ language="python",
+ symbols=[],
+ docstring=None,
+ imports=[],
+ exports=[],
+ dependencies=[],
+ dependents=[],
+ pagerank_score=0.1,
+ betweenness_score=0.0,
+ is_entry_point=False,
+ file_source_snippet="",
+ community_id=None,
+ is_api_contract=False,
+ is_test=False,
+ parse_errors=[],
+ estimated_tokens=10,
+ )
+ for f in files
+ ]
+ return assembler.assemble_module_page("Ingestion Pipeline", "python", contexts, None)
+
+
+def test_module_page_leads_with_the_concept_title(generator):
+ """The H1 is the group's name, and the directories sit under it.
+
+ A concept page covers several directories, so the old ``# Module: <path>``
+ heading had nothing honest to put after the colon. The title says what the
+ group is; the line beneath says where to go and look.
+ """
+ ctx = _module_ctx(generator._assembler, ["src/ingest/read.py", "src/parse/ast.py"])
+ page = generator._deterministic_module_page(ctx, "src/ingest", "Ingestion Pipeline", None)
+
+ assert page.title == "Ingestion Pipeline"
+ assert page.content.startswith("# Ingestion Pipeline\n")
+ assert "Module:" not in page.content
+ assert "`src/ingest`" in page.content and "`src/parse`" in page.content
+
+
+def test_module_page_of_root_files_says_so(generator):
+ """Files at the repository root have no directory to name.
+
+ ``PurePosixPath("setup.py").parent`` is ``"."``, and printing a bare dot
+ where a path belongs reads as a bug. Root-level files are a real case:
+ the grouper anchors a group there whenever a repository keeps code at the
+ top level.
+ """
+ ctx = _module_ctx(generator._assembler, ["setup.py", "main.py"])
+ page = generator._deterministic_module_page(ctx, "root", "Project Entry Points", None)
+
+ assert "Repository root" in page.content
+ assert "`.`" not in page.content
diff --git a/tests/unit/generation/test_models.py b/tests/unit/generation/test_models.py
index 0f6347185..a87c27026 100644
--- a/tests/unit/generation/test_models.py
+++ b/tests/unit/generation/test_models.py
@@ -153,8 +153,6 @@ def test_generation_config_defaults():
assert config.staleness_threshold_days == 7
assert config.expiry_threshold_days == 30
assert config.top_symbol_percentile == 0.20
- assert config.module_grouping == "curated"
- assert config.min_module_size == 3
assert config.large_file_source_pct == 0.4
assert config.reasoning == "auto"
diff --git a/tests/unit/generation/test_page_generator.py b/tests/unit/generation/test_page_generator.py
index d39dd6b7e..d0a8150af 100644
--- a/tests/unit/generation/test_page_generator.py
+++ b/tests/unit/generation/test_page_generator.py
@@ -473,10 +473,7 @@ async def test_generate_all_uses_in_memory_kg_modules_without_artifact_file():
max_tokens=256,
token_budget=100_000,
max_concurrency=2,
- module_grouping="curated",
- min_module_size=2,
coverage_pct=1.0,
- module_page_share=1.0,
dedupe_near_clones=False, # synthetic files are identical by design
)
provider = MockProvider()
diff --git a/tests/unit/generation/test_page_selection.py b/tests/unit/generation/test_page_selection.py
index 15e2efea3..db29025d6 100644
--- a/tests/unit/generation/test_page_selection.py
+++ b/tests/unit/generation/test_page_selection.py
@@ -84,37 +84,12 @@ def test_windows_target_paths_normalize() -> None:
# ---------------------------------------------------------------------------
-# Test directories do not get a module page
+# Test files do not enter the concept tree
# ---------------------------------------------------------------------------
-
-
-def _pf(path, is_test):
- from types import SimpleNamespace
-
- return SimpleNamespace(file_info=SimpleNamespace(path=path, is_test=is_test))
-
-
-def test_a_test_directory_is_not_a_module():
- from repowise.core.generation.selection.selector import _is_test_group
-
- assert _is_test_group([_pf("tests/test_a.py", True), _pf("tests/test_b.py", True)])
-
-
-def test_a_production_module_with_a_colocated_test_keeps_its_page():
- """Go and Jest put the test beside the source; that is still a module."""
- from repowise.core.generation.selection.selector import _is_test_group
-
- files = [_pf("pkg/a.go", False), _pf("pkg/b.go", False), _pf("pkg/a_test.go", True)]
- assert not _is_test_group(files)
-
-
-def test_an_even_split_is_not_a_test_group():
- from repowise.core.generation.selection.selector import _is_test_group
-
- assert not _is_test_group([_pf("pkg/a.py", False), _pf("pkg/a_test.py", True)])
-
-
-def test_empty_group_is_not_a_test_group():
- from repowise.core.generation.selection.selector import _is_test_group
-
- assert not _is_test_group([])
+#
+# The majority-vote group filter that used to live here is gone with the
+# per-directory grouping. Test files are now excluded before grouping runs,
+# one file at a time, so a production directory with a colocated test keeps
+# its page and the test file simply is not a member of it. That is asserted
+# against the real selector in
+# ``test_selection_concept_groups.py::test_test_files_never_enter_the_concept_tree``.
diff --git a/tests/unit/generation/test_selection_budget.py b/tests/unit/generation/test_selection_budget.py
index 544c0c5d4..ccf5e5df6 100644
--- a/tests/unit/generation/test_selection_budget.py
+++ b/tests/unit/generation/test_selection_budget.py
@@ -9,7 +9,6 @@ configured ``coverage_pct``. They are the contract that prevents the
from __future__ import annotations
from dataclasses import dataclass, field
-from unittest.mock import MagicMock
import pytest
@@ -24,7 +23,6 @@ from repowise.core.generation.selection.budget import (
compute_budget,
)
-
# ---------------------------------------------------------------------------
# Lightweight ParsedFile / Symbol stand-ins
# ---------------------------------------------------------------------------
@@ -179,7 +177,14 @@ def test_allocate_budget_respects_candidate_supply():
],
)
def test_select_pages_budget_tracks_coverage(coverage_pct, expected_total):
- """Total page count tracks coverage_pct within ±10%."""
+ """Budgeted page count tracks coverage_pct.
+
+ ``module_page`` is excluded from the sum on purpose. The concept groups
+ are a total partition of the production files, so that bucket is taken
+ whole rather than rationed; folding it in would assert the opposite of the
+ contract it now has. Its own invariant is asserted separately by
+ :func:`test_module_bucket_is_not_rationed_by_coverage`.
+ """
parsed, pagerank, betweenness, community = _build_synthetic_repo(958)
cfg = GenerationConfig(coverage_pct=coverage_pct)
inputs = SelectionInputs(
@@ -194,12 +199,58 @@ def test_select_pages_budget_tracks_coverage(coverage_pct, expected_total):
)
selection = select_pages(inputs)
- total = sum(selection.counts().values())
+ counts = selection.counts()
+ total = sum(v for k, v in counts.items() if k != "module_page")
tolerance = max(20, int(expected_total * 0.20))
- assert abs(total - expected_total) <= tolerance, (
- f"coverage={coverage_pct}: expected ~{expected_total}, got {total}"
- )
+ assert (
+ abs(total - expected_total) <= tolerance
+ ), f"coverage={coverage_pct}: expected ~{expected_total}, got {total}"
+
+
+def test_module_bucket_is_not_rationed_by_coverage():
+ """Every concept group is emitted, whatever the coverage setting.
+
+ The partition is total, so a rationed module bucket does not make a
+ smaller wiki — it makes one with production files that belong to no page,
+ and the tree stops being a map of the repository. Asserted across the
+ coverage range because the budget is what used to truncate it.
+ """
+ parsed, pagerank, betweenness, community = _build_synthetic_repo(958)
+
+ def _selection_for(pct: float):
+ return select_pages(
+ SelectionInputs(
+ parsed_files=parsed,
+ pagerank=pagerank,
+ betweenness=betweenness,
+ community=community,
+ community_info=None,
+ sccs=[],
+ git_meta_map=None,
+ config=GenerationConfig(coverage_pct=pct),
+ )
+ )
+
+ selections = {pct: _selection_for(pct) for pct in (0.10, 0.20, 0.50)}
+ group_counts = {pct: len(s.module_groups) for pct, s in selections.items()}
+
+ assert (
+ len(set(group_counts.values())) == 1
+ ), f"coverage changed the concept page count: {group_counts}"
+ assert all(c > 0 for c in group_counts.values())
+
+ # The allocation must agree with what was emitted, or the cost estimator
+ # quotes a number the run does not honour.
+ for pct, sel in selections.items():
+ assert sel.allocation is not None
+ assert sel.allocation.module_page == len(sel.module_groups), pct
+
+ # Totality: every production file the grouper was given lands on exactly
+ # one page. This is the property rationing used to break.
+ for pct, sel in selections.items():
+ claimed = [p for g in sel.module_groups for p in g.file_paths]
+ assert len(claimed) == len(set(claimed)), f"double-claimed at {pct}"
def test_select_pages_emits_no_pages_for_empty_repo():
@@ -283,15 +334,20 @@ def test_landmarks_are_pulled_into_budget_without_inflating_count():
def test_landmarks_respect_coverage_across_sizes():
- """The total page count still tracks coverage_pct with landmarks active."""
+ """The budgeted page count still tracks coverage_pct with landmarks active.
+
+ ``module_page`` is excluded for the same reason as
+ :func:`test_select_pages_budget_tracks_coverage`: the concept bucket is
+ taken whole, so it does not answer to the coverage budget.
+ """
for n, pct in [(40, 0.10), (500, 0.20), (1200, 0.10)]:
parsed, pr, bet, comm = _build_synthetic_repo(n, entry_points=5)
sel = _select(parsed, pr, bet, comm, pct)
- total = sum(sel.counts().values())
+ total = sum(v for k, v in sel.counts().items() if k != "module_page")
# Landmark pull-in displaces rather than adds, so the total stays in
# the same ±20% band the budget contract guarantees.
expected = compute_budget(n, pct)
tolerance = max(20, int(expected * 0.30))
- assert abs(total - expected) <= tolerance, (
- f"n={n} pct={pct}: expected ~{expected}, got {total}"
- )
+ assert (
+ abs(total - expected) <= tolerance
+ ), f"n={n} pct={pct}: expected ~{expected}, got {total}"
diff --git a/tests/unit/generation/test_selection_concept_groups.py b/tests/unit/generation/test_selection_concept_groups.py
new file mode 100644
index 000000000..b1c197c0c
--- /dev/null
+++ b/tests/unit/generation/test_selection_concept_groups.py
@@ -0,0 +1,309 @@
+"""Concept grouping in the selection layer.
+
+Replaces the curated/community/top-dir grouping tests. That grouping is gone,
+not deprecated: ``module_page`` is now produced from the concept partition
+(``concept_tree.grouping.group_files``), so the properties worth asserting
+changed from "which of three sources won" to the two the partition
+guarantees — every production file is covered exactly once, and the same
+input always produces the same page identities.
+
+Fixtures deliberately disagree with the alphabet where ordering matters, and
+every path fixture includes a root-level file, because the repository root's
+directory path is the empty string and a falsy check cannot tell it from "no
+directory found". That defect shipped once.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+from repowise.core.generation.models import GenerationConfig
+from repowise.core.generation.selection import SelectionInputs, select_pages
+from repowise.core.generation.selection.selector import _build_module_groups
+from tests.unit.generation.test_selection_budget import (
+ FakeFileInfo,
+ FakeParsedFile,
+ FakeSymbol,
+)
+
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def _paths() -> list[str]:
+ """Production paths spanning several subtrees, plus root-level files.
+
+ PageRank below is assigned so the highest-scoring group is NOT the one
+ that sorts first alphabetically, so an ordering assertion cannot pass by
+ accident on a sorted-by-path implementation.
+ """
+ return (
+ [f"packages/app/src/core/ingestion/f{i}.py" for i in range(8)]
+ + [f"packages/app/src/core/analysis/f{i}.py" for i in range(6)]
+ + [f"packages/app/src/ui/c4/f{i}.py" for i in range(5)]
+ + ["packages/app/src/ui/tiny/f0.py", "packages/app/src/ui/tiny/f1.py"]
+ # Root-level files: the empty-string directory is real.
+ + ["setup.py", "main.py"]
+ )
+
+
+def _test_paths() -> list[str]:
+ return [f"tests/unit/test_f{i}.py" for i in range(6)]
+
+
+def _inputs(
+ *,
+ with_tests: bool = True,
+ kg_modules: list[dict] | None = None,
+ paths: list[str] | None = None,
+):
+ prod = paths if paths is not None else _paths()
+ paths = prod + (_test_paths() if with_tests else [])
+ parsed = [
+ FakeParsedFile(
+ file_info=FakeFileInfo(path=p, is_test=p.startswith("tests/")),
+ symbols=[FakeSymbol(name="fn")],
+ )
+ for p in paths
+ ]
+ # ui/c4 carries the highest mass, and "packages/app/src/ui/c4" sorts after
+ # "packages/app/src/core/...", so score-descending order must not equal
+ # path order.
+ pagerank = {p: (1.0 if "/ui/c4/" in p else 0.1) for p in paths}
+ return SelectionInputs(
+ parsed_files=parsed,
+ pagerank=pagerank,
+ betweenness={p: 0.0 for p in paths},
+ community={p: 0 for p in paths},
+ community_info=None,
+ sccs=[],
+ git_meta_map=None,
+ config=GenerationConfig(coverage_pct=0.20),
+ kg_modules=kg_modules,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Coverage and totality
+# ---------------------------------------------------------------------------
+
+
+def test_every_production_file_is_covered_exactly_once():
+ groups = [g for _, g in _build_module_groups(_inputs())]
+ claimed = [p for g in groups for p in g.file_paths]
+
+ assert sorted(claimed) == sorted(_paths()), "the partition is not total"
+ assert len(claimed) == len(set(claimed)), "a file was claimed by two pages"
+
+
+def test_root_level_files_get_a_usable_target_path():
+ """The repository root is a real directory whose path is ``""``.
+
+ A group anchored there must not persist an empty ``target_path``: the page
+ id is ``"{type}:{target_path}"``, so an empty target mints ``module_page:``
+ and leaves the page with nothing for the tree, the breadcrumbs or the
+ bench gold set to match on.
+ """
+ groups = [g for _, g in _build_module_groups(_inputs())]
+ owning = [g for g in groups if "setup.py" in g.file_paths]
+
+ assert owning, "root-level files were dropped from the partition"
+ assert any("main.py" in g.file_paths for g in groups)
+ for g in groups:
+ assert g.key, "a group persisted an empty target_path"
+ assert any(
+ g.key == "root" for g in owning
+ ), f"root-anchored group did not get the root target: {[g.key for g in owning]}"
+
+
+def test_test_files_never_enter_the_concept_tree():
+ """D8: tests keep file pages, they do not get concept pages."""
+ groups = [g for _, g in _build_module_groups(_inputs())]
+ claimed = {p for g in groups for p in g.file_paths}
+
+ assert not any(p.startswith("tests/") for p in claimed)
+ # And their absence is not an artifact of them being absent from input.
+ assert any(p.file_info.path.startswith("tests/") for p in _inputs().parsed_files)
+
+
+def test_dropping_the_test_files_does_not_change_the_partition():
+ """Excluding tests must be a filter, not a force that reshapes groups."""
+ with_tests = [g.structural_key for _, g in _build_module_groups(_inputs())]
+ without = [g.structural_key for _, g in _build_module_groups(_inputs(with_tests=False))]
+
+ assert with_tests == without
+
+
+# ---------------------------------------------------------------------------
+# Identity
+# ---------------------------------------------------------------------------
+
+
+def test_groups_carry_a_concept_prefixed_structural_key():
+ """The prefix is what lets a wiki tell a concept page from an old one."""
+ groups = [g for _, g in _build_module_groups(_inputs())]
+
+ assert groups
+ assert all(g.structural_key.startswith("concept-") for g in groups)
+ keys = [g.structural_key for g in groups]
+ assert len(keys) == len(set(keys)), "two groups share an identity"
+
+
+def test_target_paths_are_unique():
+ """The page id is derived from the target path alone.
+
+ Two groups resolving to one target is a row collision: the second page
+ silently overwrites the first on persist.
+ """
+ groups = [g for _, g in _build_module_groups(_inputs())]
+ keys = [g.key for g in groups]
+
+ assert len(keys) == len(set(keys)), f"duplicate target paths: {keys}"
+
+
+def test_identity_is_stable_across_processes():
+ """Hash randomisation must not move a page's identity.
+
+ Run in two subprocesses under different ``PYTHONHASHSEED`` values. Calling
+ the function twice in one process cannot fail this way and therefore
+ cannot test it.
+ """
+ script = textwrap.dedent(
+ """
+ from tests.unit.generation.test_selection_concept_groups import (
+ _build_module_groups, _inputs,
+ )
+ groups = [g for _, g in _build_module_groups(_inputs())]
+ print("|".join(f"{g.key}={g.structural_key}" for g in groups))
+ """
+ )
+ outs = []
+ for seed in ("0", "12345"):
+ env = {**os.environ, "PYTHONHASHSEED": seed}
+ proc = subprocess.run(
+ [sys.executable, "-c", script],
+ capture_output=True,
+ text=True,
+ env=env,
+ cwd=str(_REPO_ROOT),
+ timeout=120,
+ )
+ assert proc.returncode == 0, proc.stderr
+ outs.append(proc.stdout.strip())
+
+ assert outs[0] == outs[1], "page identity moved with the hash seed"
+ assert outs[0], "the subprocess produced no groups, so this asserted nothing"
+
+
+# ---------------------------------------------------------------------------
+# Ordering and naming
+# ---------------------------------------------------------------------------
+
+
+def test_ranked_by_summed_pagerank_not_by_path():
+ """The heaviest subsystem sorts first even though its path sorts last."""
+ scored = _build_module_groups(_inputs())
+ ordered_keys = [g.key for _, g in scored]
+
+ assert len(scored) > 1, "a one-group fixture cannot test ordering"
+ # The fixture is only meaningful if path order disagrees with score order,
+ # or the assertion below would pass on a sorted-by-path implementation.
+ assert ordered_keys != sorted(
+ ordered_keys
+ ), f"fixture is degenerate: score order equals path order ({ordered_keys})"
+ # ui/c4 carries 1.0 per file against 0.1 elsewhere, so whichever group
+ # holds it must come first.
+ top_group = scored[0][1]
+ assert any(
+ "/ui/c4/" in p for p in top_group.file_paths
+ ), f"expected the ui/c4 mass to rank first, got {top_group.key}"
+
+
+def test_display_is_a_name_not_a_bare_path():
+ groups = [g for _, g in _build_module_groups(_inputs())]
+
+ for g in groups:
+ assert g.display, f"{g.key} has no title"
+ assert "/" not in g.display, f"{g.display} is a path, not a name"
+
+
+def test_layer_labels_from_the_kg_reach_the_title():
+ """The KG layer map is read when present and absent-safe when not."""
+ modules = [
+ {
+ "id": "module:ui-c4",
+ "path": "packages/app/src/ui/c4",
+ "layerId": "layer:presentation",
+ "nodeIds": [f"file:{p}" for p in _paths() if "/ui/c4/" in p],
+ }
+ ]
+ with_kg = [g for _, g in _build_module_groups(_inputs(kg_modules=modules))]
+ without_kg = [g for _, g in _build_module_groups(_inputs())]
+
+ # Membership is unchanged by the layer signal: it steers merging of
+ # adjacent runs, it never forces a split.
+ assert [g.structural_key for g in with_kg] == [g.structural_key for g in without_kg]
+ assert all(g.display for g in with_kg)
+
+ # The label itself has to reach a title, or the map is being read and
+ # thrown away. Titles differ only where the layer applies: the prefix
+ # exists to rescue a name too short to say anything, so groups whose
+ # paths already yield two words are identical either way.
+ titled = [g.display for g in with_kg]
+ untitled = [g.display for g in without_kg]
+ assert titled != untitled, titled
+ assert any("Presentation" in t for t in titled), titled
+
+
+# ---------------------------------------------------------------------------
+# The bucket is not rationed
+# ---------------------------------------------------------------------------
+
+
+def test_select_pages_emits_every_group():
+ inputs = _inputs()
+ groups = _build_module_groups(inputs)
+ selection = select_pages(inputs)
+
+ assert len(selection.module_groups) == len(groups)
+ assert selection.allocation is not None
+ assert selection.allocation.module_page == len(groups)
+
+
+def test_structural_key_survives_selection():
+ """The key the grouper minted has to reach the page, not be recomputed."""
+ selection = select_pages(_inputs())
+
+ assert selection.module_groups
+ assert all(g.structural_key.startswith("concept-") for g in selection.module_groups)
+
+
+# ---------------------------------------------------------------------------
+# Titles a reader can tell apart
+# ---------------------------------------------------------------------------
+
+
+def test_two_groups_never_share_a_title():
+ """Same directory name in two packages must not produce two identical rows.
+
+ Page identity is structural, so the pages stay distinct — but the tree
+ shows the title, and two rows reading "Ui Components" are the same row as
+ far as a reader is concerned.
+ """
+ paths = (
+ [f"packages/alpha/src/ui/f{i}.py" for i in range(8)]
+ + [f"packages/beta/src/ui/f{i}.py" for i in range(8)]
+ + ["setup.py"]
+ )
+ groups = [g for _, g in _build_module_groups(_inputs(paths=paths))]
+
+ titles = [g.display for g in groups]
+ assert len(titles) == len(set(titles)), titles
+ assert len({g.key for g in groups}) == len(groups)
diff --git a/tests/unit/generation/test_selection_curated_modules.py b/tests/unit/generation/test_selection_curated_modules.py
deleted file mode 100644
index 43ff6c195..000000000
--- a/tests/unit/generation/test_selection_curated_modules.py
+++ /dev/null
@@ -1,278 +0,0 @@
-"""Curated module grouping in the selection layer.
-
-Covers the ``module_grouping="curated"`` source (key = real dir path,
-display = curated name, cohesion = None, Σ-PageRank ranking), every row of
-the fallback matrix, and the golden inertness contract: with the default
-``"community"`` grouping, passing ``kg_modules`` changes NOTHING — the
-ship-dark guarantee that every existing path stays byte-identical.
-"""
-
-from __future__ import annotations
-
-from repowise.core.generation.models import GenerationConfig
-from repowise.core.generation.selection import SelectionInputs, select_pages
-from repowise.core.generation.selection.selector import _build_module_groups
-from tests.unit.generation.test_selection_budget import (
- FakeFileInfo,
- FakeParsedFile,
- FakeSymbol,
- _build_synthetic_repo,
-)
-
-# ---------------------------------------------------------------------------
-# Fixtures
-# ---------------------------------------------------------------------------
-
-
-def _repo_with_modules():
- """A synthetic repo plus a matching curated-modules artifact."""
- paths = (
- [f"packages/app/src/core/ingestion/f{i}.py" for i in range(8)]
- + [f"packages/app/src/core/analysis/f{i}.py" for i in range(6)]
- + [f"packages/app/src/ui/c4/f{i}.py" for i in range(5)]
- + ["packages/app/src/ui/tiny/f0.py", "packages/app/src/ui/tiny/f1.py"]
- )
- parsed = [
- FakeParsedFile(file_info=FakeFileInfo(path=p), symbols=[FakeSymbol(name="fn")])
- for p in paths
- ]
- # ui/c4 members carry the highest PageRank so Σ-PageRank must rank that
- # module first despite it being smaller than core/ingestion.
- pagerank = {p: (1.0 if "/ui/c4/" in p else 0.1) for p in paths}
- community = {p: (0 if "/core/" in p else 1) for p in paths}
-
- modules = [
- {
- "id": "module:packages-app-src-core-ingestion",
- "name": "core/ingestion",
- "path": "packages/app/src/core/ingestion",
- "layerId": "layer:service",
- "nodeIds": [f"file:{p}" for p in paths if "/core/ingestion/" in p],
- "language": "python",
- },
- {
- "id": "module:packages-app-src-core-analysis",
- "name": "core/analysis",
- "path": "packages/app/src/core/analysis",
- "layerId": "layer:service",
- "nodeIds": [f"file:{p}" for p in paths if "/core/analysis/" in p],
- "language": "python",
- },
- {
- "id": "module:packages-app-src-ui-c4",
- "name": "ui/c4",
- "path": "packages/app/src/ui/c4",
- "layerId": "layer:ui",
- "nodeIds": [f"file:{p}" for p in paths if "/ui/c4/" in p],
- "language": "python",
- },
- {
- # Below min_module_size=3 → must be skipped, never a page.
- "id": "module:packages-app-src-ui-tiny",
- "name": "ui/tiny",
- "path": "packages/app/src/ui/tiny",
- "layerId": "layer:ui",
- "nodeIds": [f"file:{p}" for p in paths if "/ui/tiny/" in p],
- "language": "python",
- },
- ]
- return parsed, pagerank, community, modules
-
-
-def _inputs(parsed, pagerank, community, *, cfg=None, kg_modules=None):
- return SelectionInputs(
- parsed_files=parsed,
- pagerank=pagerank,
- betweenness=dict.fromkeys(pagerank, 0.0),
- community=community,
- community_info=None,
- sccs=[],
- git_meta_map=None,
- config=cfg or GenerationConfig(),
- kg_modules=kg_modules,
- )
-
-
-def _cfg(grouping: str) -> GenerationConfig:
- return GenerationConfig(module_grouping=grouping)
-
-
-# ---------------------------------------------------------------------------
-# Curated grouping source
-# ---------------------------------------------------------------------------
-
-
-class TestCuratedGrouping:
- def test_groups_keyed_by_dir_path_with_curated_names(self):
- parsed, pr, comm, modules = _repo_with_modules()
- groups = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)
- )
- by_key = {g.key: g for _, g in groups}
- assert "packages/app/src/core/ingestion" in by_key
- g = by_key["packages/app/src/core/ingestion"]
- assert g.display == "core/ingestion"
- assert g.label == "core/ingestion"
- assert g.cohesion is None
- assert g.language == "python"
- assert len(g.file_paths) == 8
-
- def test_ranked_by_sum_pagerank_of_members(self):
- parsed, pr, comm, modules = _repo_with_modules()
- groups = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)
- )
- # ui/c4: 5 x 1.0 = 5.0 beats core/ingestion: 8 x 0.1 = 0.8.
- assert groups[0][1].display == "ui/c4"
- scores = [s for s, _ in groups]
- assert scores == sorted(scores, reverse=True)
-
- def test_min_module_size_floor_kills_tiny_modules(self):
- parsed, pr, comm, modules = _repo_with_modules()
- groups = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)
- )
- assert "packages/app/src/ui/tiny" not in {g.key for _, g in groups}
-
- def test_members_restricted_to_parsed_code_files(self):
- parsed, pr, comm, modules = _repo_with_modules()
- modules[0]["nodeIds"].append("file:packages/app/src/core/ingestion/ghost.py")
- groups = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)
- )
- by_key = {g.key: g for _, g in groups}
- assert not any(
- "ghost" in p
- for p in by_key["packages/app/src/core/ingestion"].file_paths
- )
-
- def test_no_community_ids_in_curated_keys(self):
- parsed, pr, comm, modules = _repo_with_modules()
- groups = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)
- )
- assert not any(g.key.startswith("community-") for _, g in groups)
-
-
-# ---------------------------------------------------------------------------
-# Fallback matrix (one test per row)
-# ---------------------------------------------------------------------------
-
-
-class TestFallbackMatrix:
- def test_curated_without_artifact_falls_back_to_community(self):
- """curation off / degraded / no KG json → today's community path."""
- parsed, pr, _bet, comm = _build_synthetic_repo(100)
- curated_no_artifact = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=None)
- )
- community = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("community"))
- )
- assert curated_no_artifact == community
-
- def test_curated_with_all_modules_below_floor_emits_none_not_community(self):
- """A present-but-filtered artifact must not mix vocabularies."""
- parsed, pr, comm, modules = _repo_with_modules()
- tiny_only = [m for m in modules if m["name"] == "ui/tiny"]
- groups = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=tiny_only)
- )
- assert groups == []
-
- def test_explicit_community_honored_even_with_artifact(self):
- parsed, pr, comm, modules = _repo_with_modules()
- with_artifact = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("community"), kg_modules=modules)
- )
- without = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("community"))
- )
- assert with_artifact == without
- assert not any(g.key.startswith("packages/") for _, g in with_artifact)
-
- def test_explicit_top_dir_honored_even_with_artifact(self):
- parsed, pr, comm, modules = _repo_with_modules()
- with_artifact = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("top_dir"), kg_modules=modules)
- )
- without = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("top_dir"))
- )
- assert with_artifact == without
- assert {g.key for _, g in with_artifact} == {"packages"}
-
-
-# ---------------------------------------------------------------------------
-# Golden inertness — the ship-dark contract
-# ---------------------------------------------------------------------------
-
-
-class TestGoldenByteIdentity:
- def test_community_selection_identical_with_and_without_kg_modules(self):
- """With the "community" escape hatch, kg_modules must change NOTHING.
-
- This was the ship-dark rollout guarantee while "community" was the
- default; post-flip it pins the revert story — setting
- module_grouping="community" restores the pre-curation selection
- byte-for-byte regardless of what the KG artifact carries.
- """
- parsed, pr, bet, comm = _build_synthetic_repo(300)
- _, _, _, modules = _repo_with_modules()
- cfg = GenerationConfig(module_grouping="community")
-
- def run(kg_modules):
- return select_pages(
- SelectionInputs(
- parsed_files=parsed,
- pagerank=pr,
- betweenness=bet,
- community=comm,
- community_info=None,
- sccs=[],
- git_meta_map=None,
- config=cfg,
- kg_modules=kg_modules,
- )
- )
-
- assert run(None) == run(modules)
-
- def test_curated_selection_is_deterministic(self):
- parsed, pr, comm, modules = _repo_with_modules()
- cfg = _cfg("curated")
-
- def run():
- return select_pages(
- _inputs(parsed, pr, comm, cfg=cfg, kg_modules=modules)
- )
-
- a, b = run(), run()
- assert a.module_groups == b.module_groups
- assert a == b
-
-
-class TestWholeLayerDedupe:
- """Modules 1:1 with a layer never get a module page (layer_page covers them)."""
-
- def test_whole_layer_module_skipped(self):
- parsed, pr, comm, modules = _repo_with_modules()
- modules = [dict(m) for m in modules]
- modules[0]["wholeLayer"] = True # core/ingestion now 1:1 with its layer
- groups = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)
- )
- keys = {g.key for _, g in groups}
- assert "packages/app/src/core/ingestion" not in keys
- assert "packages/app/src/ui/c4" in keys
-
- def test_all_whole_layer_yields_no_module_pages_not_community(self):
- # Flat-lib shape: every module is 1:1 with a layer → zero module
- # pages, and NO fallback to community grouping (vocabulary stays
- # curated; the layer pages carry the documentation).
- parsed, pr, comm, modules = _repo_with_modules()
- modules = [dict(m, wholeLayer=True) for m in modules]
- groups = _build_module_groups(
- _inputs(parsed, pr, comm, cfg=_cfg("curated"), kg_modules=modules)
- )
- assert groups == []
diff --git a/tests/unit/generation/test_templates.py b/tests/unit/generation/test_templates.py
index 6916b0b87..31ad63c29 100644
--- a/tests/unit/generation/test_templates.py
+++ b/tests/unit/generation/test_templates.py
@@ -114,7 +114,7 @@ def test_file_page_contains_file_path(jinja_env, file_page_ctx):
@pytest.fixture(scope="module")
def module_page_ctx() -> ModulePageContext:
return ModulePageContext(
- module_path="python_pkg",
+ title="Calculation Engine",
language="python",
total_symbols=5,
public_symbols=3,
@@ -123,6 +123,7 @@ def module_page_ctx() -> ModulePageContext:
dependents=[],
pagerank_mean=0.3,
files=["python_pkg/calculator.py", "python_pkg/models.py"],
+ directories=["python_pkg"],
)
@@ -136,9 +137,16 @@ def test_module_page_has_heading(jinja_env, module_page_ctx):
assert "##" in result
-def test_module_page_contains_module_path(jinja_env, module_page_ctx):
+def test_module_page_names_the_group_and_where_it_lives(jinja_env, module_page_ctx):
+ """The title is prose and the directories are the path. Both must appear.
+
+ A concept page spans directories, so a reader who only gets the title has
+ no way to go and look, and one who only gets a path has to guess what the
+ group is for.
+ """
result = render(jinja_env, "module_page.j2", module_page_ctx)
- assert module_page_ctx.module_path in result
+ assert module_page_ctx.title in result
+ assert "python_pkg" in result
# ---------------------------------------------------------------------------
@@ -339,5 +347,3 @@ def test_scc_page_has_heading(jinja_env, scc_page_ctx):
def test_scc_page_contains_cycle_description(jinja_env, scc_page_ctx):
result = render(jinja_env, "scc_page.j2", scc_page_ctx)
assert scc_page_ctx.cycle_description in result
-
-
diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.py
index 6d8fc788d..4bbc2e6dc 100644
--- a/tests/unit/ingestion/test_traverser.py
+++ b/tests/unit/ingestion/test_traverser.py
@@ -781,3 +781,56 @@ class TestEntryPointFlag:
flagged = self._flagged(tmp_path)
assert "cli.py" in flagged
assert "pkg/cli.py" not in flagged
+
+
+# ---------------------------------------------------------------------------
+# Test-file classification
+# ---------------------------------------------------------------------------
+
+
+ "rel_path",
+ [
+ "tests/conftest.py",
+ "tests/type_check/typing_app.py",
+ "packages/ui/__tests__/brand.test.ts",
+ "packages/ui/__tests__/chat/panel.tsx",
+ "src/components/Button.test.tsx",
+ "src/lib/parse.spec.ts",
+ "test/fixtures/app.py",
+ "spec/support/helper.rb",
+ "src/pkg/tests/helpers.py",
+ "pkg/test_thing.py",
+ "pkg/thing_test.go",
+ ],
+)
+def test_test_files_are_recognised_including_root_level_suites(rel_path):
+ """Directory and filename conventions across languages, not just Python.
+
+ Two gaps, both of which let test files into the concept tree. ``"/tests/"
+ in path`` needs a slash on both sides, so a root-level ``tests/`` — the
+ usual layout — read as production code. And the stem rule knew only
+ ``test_x`` / ``x_test``, so the whole JavaScript convention
+ (``__tests__/`` directories, ``x.test.ts`` filenames) was invisible:
+ measured on this repository, 122 test files survived into the production
+ set and five concept pages were nothing but tests.
+ """
+ from repowise.core.ingestion.traverser import _is_test_file
+
+ assert _is_test_file(rel_path, Path(rel_path).name)
+
+
+ "rel_path",
+ [
+ "src/latest/api.py",
+ "src/contest/rules.py",
+ "protests/model.py",
+ "src/testing_utils.py",
+ ],
+)
+def test_production_paths_that_merely_contain_the_word_are_not_tests(rel_path):
+ """Segment match, not substring: ``contest`` and ``latest`` are not tests."""
+ from repowise.core.ingestion.traverser import _is_test_file
+
+ assert not _is_test_file(rel_path, Path(rel_path).name)
diff --git a/tests/unit/pipeline/test_authoritative_page_types.py b/tests/unit/pipeline/test_authoritative_page_types.py
new file mode 100644
index 000000000..c4937c701
--- /dev/null
+++ b/tests/unit/pipeline/test_authoritative_page_types.py
@@ -0,0 +1,122 @@
+"""What a generating run claims to have decided, and therefore what the sweep may retire.
+
+``authoritative_page_types`` is the signal ``_sweep_stale_generated_pages``
+reads before deleting prior rows of a page type the run did not reproduce. Get
+it wrong in the permissive direction and a degraded run wipes good pages; get
+it wrong in the strict direction and superseded pages accumulate forever.
+
+``module_page`` is the interesting one. The concept grouping partitions every
+production file, takes no optional input, and is not rationed by the coverage
+budget, so a run that produced any concept page produced all of them. It used
+to be declared authoritative only when a curated knowledge-graph artifact was
+present, which left exactly the KG-less run, the one carrying the most stale
+rows, unable to clean up after itself.
+
+The claim is still made on evidence rather than on "generation ran", because
+there are ways to emit zero concept pages that do not mean "there is nothing
+to document" — ``file_pages_only`` returns before that level, and a provider
+error on it is logged per page rather than raised.
+"""
+
+from __future__ import annotations
+
+from repowise.core.pipeline import run_pipeline
+from repowise.core.providers.llm.mock import MockProvider
+
+
+def _write_repo(root):
+ root.mkdir()
+ (root / "pkg").mkdir()
+ (root / "pkg" / "main.py").write_text(
+ "def hello() -> str:\n return 'hello'\n",
+ encoding="utf-8",
+ )
+ (root / "pkg" / "util.py").write_text(
+ "def helper(n: int) -> int:\n return n + 1\n",
+ encoding="utf-8",
+ )
+ return root
+
+
+async def test_generating_run_is_authoritative_for_module_pages(tmp_path):
+ repo_path = _write_repo(tmp_path / "repo")
+
+ result = await run_pipeline(
+ repo_path,
+ generate_docs=True,
+ llm_client=MockProvider(),
+ concurrency=1,
+ test_run=True,
+ )
+
+ assert result.generated_pages
+ assert "module_page" in result.authoritative_page_types
+
+
+async def test_a_run_that_does_not_generate_claims_nothing(tmp_path):
+ """No pages produced means no page type was decided.
+
+ An index-only run has no opinion about which concept pages should exist,
+ so letting it claim authority would let it retire every one of them.
+ """
+ repo_path = _write_repo(tmp_path / "repo")
+
+ result = await run_pipeline(
+ repo_path,
+ generate_docs=False,
+ concurrency=1,
+ test_run=True,
+ )
+
+ assert not result.authoritative_page_types
+
+
+async def test_a_run_that_never_reaches_the_concept_level_claims_nothing(tmp_path):
+ """``file_pages_only`` stops before concept pages exist.
+
+ Claiming authority there would let a file-pages-only run delete every
+ concept page in the store, which is the most destructive shape this
+ signal has.
+ """
+ from repowise.core.generation.models import GenerationConfig
+
+ repo_path = _write_repo(tmp_path / "repo")
+
+ result = await run_pipeline(
+ repo_path,
+ generate_docs=True,
+ llm_client=MockProvider(),
+ concurrency=1,
+ test_run=True,
+ generation_config=GenerationConfig(file_pages_only=True),
+ )
+
+ assert result.generated_pages
+ assert not any(p.page_type == "module_page" for p in result.generated_pages)
+ assert "module_page" not in result.authoritative_page_types
+
+
+async def test_a_repository_with_no_production_code_still_claims(tmp_path):
+ """Zero pages because there is nothing to group is the case the sweep wants.
+
+ A repository whose code was deleted, or that is only tests, should have
+ its concept pages retired rather than kept forever behind an
+ "it produced nothing, so trust nothing" rule.
+ """
+ repo_path = tmp_path / "repo"
+ repo_path.mkdir()
+ (repo_path / "tests").mkdir()
+ (repo_path / "tests" / "test_thing.py").write_text(
+ "def test_x():\n assert True\n", encoding="utf-8"
+ )
+
+ result = await run_pipeline(
+ repo_path,
+ generate_docs=True,
+ llm_client=MockProvider(),
+ concurrency=1,
+ test_run=True,
+ )
+
+ assert not any(p.page_type == "module_page" for p in (result.generated_pages or []))
+ assert "module_page" in result.authoritative_page_types