Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): derive a concept outline deterministically, name it with an LLM (#1023)

Raghav Chamadiya5aac96ea71e6
diff --git a/packages/core/src/repowise/core/generation/concept_tree/__init__.py b/packages/core/src/repowise/core/generation/concept_tree/__init__.py
new file mode 100644
index 000000000..ff716834b
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/concept_tree/__init__.py
@@ -0,0 +1,30 @@
+"""Derive a repo-specific concept outline: deterministic grouping, LLM naming.
+
+The split is not a style preference, it is what the probes measured. Feeding a
+planner the whole directory inventory and asking it to both allocate files and
+name the result gave 91.7% coverage on one run and 87.5% on the next from an
+identical payload, and pushing harder on allocation dropped coverage to 88.2%
+while raising fabricated paths ninefold. Membership decisions are the part an
+LLM is unreliable at and a bounded-partition function is exact at; naming is
+the part it is genuinely good at.
+
+So :mod:`grouping` decides which files each page covers, with no model in the
+loop, and :mod:`naming` decides what those pages are called. Coverage becomes
+100% by construction rather than by luck, and the identity of a page stops
+depending on a sampling temperature.
+"""
+
+from .grouping import ConceptGroup, GroupingParams, group_files
+from .models import ConceptOutline, ConceptPage, ConceptSection, OutlineReport
+from .validation import validate_outline
+
+__all__ = [
+ "ConceptGroup",
+ "ConceptOutline",
+ "ConceptPage",
+ "ConceptSection",
+ "GroupingParams",
+ "OutlineReport",
+ "group_files",
+ "validate_outline",
+]
diff --git a/packages/core/src/repowise/core/generation/concept_tree/grouping.py b/packages/core/src/repowise/core/generation/concept_tree/grouping.py
new file mode 100644
index 000000000..d6c0b967e
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/concept_tree/grouping.py
@@ -0,0 +1,445 @@
+"""Bin a repository's production files into bounded, path-local groups.
+
+One page per directory gives ``analysis/dead_code`` a page whether it earns one
+and ``core/generation`` exactly one whether it needs three. One page per LLM
+whim gives a different answer every run. This gives a third thing: a partition
+of the directory tree into subtrees of bounded size, computed from the tree
+itself.
+
+The rule is a single bottom-up pass. A directory whose whole subtree fits under
+the ceiling becomes one group and stops recursing, which is what keeps a page
+about a place rather than about an arbitrary slice of one. A directory too big
+to fit descends, and the pieces its children could not fill on their own are
+accumulated in path order and flushed when they would overflow. Path locality
+therefore falls out of the traversal rather than being asked for.
+
+Three properties this has to hold, all of them load-bearing elsewhere:
+
+* **Deterministic.** Children are visited in sorted order and every tie breaks
+ on a path string, so the same file set always produces the same groups. Page
+ identity hashes the member list (D2), so a grouping that drifted between runs
+ would mint new page ids on every index and strand the old rows.
+* **Total.** Every non-test production file lands in exactly one group. The
+ validator asserts this rather than trusting it, but it is true by
+ construction: the recursion partitions the tree and never drops a branch.
+* **Local.** A group is a subtree or a run of adjacent siblings, never a
+ scattering. That is what lets a group keep a real directory in
+ ``target_path``, which the bench gold set matches on.
+
+The layer signal steers which adjacent runs merge; it does not force a split.
+A subtree small enough to fit stays one group even when it spans two layers,
+because recursion stops as soon as a subtree fits. Splitting on layer wherever
+layers disagree would fragment the outline and would inherit the layer data's
+own noise, and an occasionally impure small page is the better trade.
+
+The size ceiling comes from a coarse ladder keyed on repository size rather
+than from a division, because a ceiling computed as ``total / target`` moves
+whenever any file is added and would reshuffle the whole tree on a one-line
+commit. The ladder only moves at a band edge.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+
+from ..models import member_structural_key
+
+#: Prefix for a concept group's structural key. Distinct from ``module`` and
+#: ``scc`` so a key can never be mistaken for another page type's.
+STRUCTURAL_KEY_PREFIX = "concept"
+
+
+@dataclass(frozen=True)
+class GroupingParams:
+ """Size bounds for the partition.
+
+ ``max_files`` is the ceiling a single group may reach; ``min_files`` is the
+ size below which a group prefers to be absorbed by a neighbour rather than
+ stand alone. ``min_files`` is advisory — a repository can contain a single
+ isolated directory smaller than it — while ``max_files`` is enforced except
+ for one unavoidable case: a single directory holding more files than the
+ ceiling cannot be split any further by a path-based rule, and splitting it
+ on some other axis would break locality. Those are reported, not hidden.
+ """
+
+ min_files: int = 10
+ max_files: int = 30
+
+
+# Repository size bands and the ceiling each one uses. Chosen so a repository
+# of a few hundred files lands near 50 pages and one of a few thousand lands
+# near 80, rather than growing without bound. Bands are wide on purpose: the
+# ceiling must not move when a commit adds a file, because every group's
+# identity would move with it.
+_SIZE_LADDER: tuple[tuple[int, GroupingParams], ...] = (
+ (600, GroupingParams(min_files=6, max_files=14)),
+ (1200, GroupingParams(min_files=8, max_files=22)),
+ (2400, GroupingParams(min_files=10, max_files=28)),
+ (4800, GroupingParams(min_files=14, max_files=52)),
+ (10_000, GroupingParams(min_files=20, max_files=95)),
+)
+
+_LADDER_TOP = GroupingParams(min_files=32, max_files=160)
+
+
+def params_for(file_count: int) -> GroupingParams:
+ """The size bounds a repository of *file_count* production files uses."""
+ for ceiling, params in _SIZE_LADDER:
+ if file_count <= ceiling:
+ return params
+ return _LADDER_TOP
+
+
+@dataclass
+class ConceptGroup:
+ """The files one concept page will cover.
+
+ ``target_path`` is a real directory and stays one. It is the page id
+ (``compute_page_id`` is ``"{type}:{target_path}"``) and it is what the
+ bench gold set matches by exact equality, so a group that merged several
+ directories still has to name one of them. ``structural_key`` is where the
+ abstract identity lives — a hash of the members, which is the only thing
+ that actually says which page this is.
+ """
+
+ members: list[str]
+ #: Directories fully contained in this group, sorted. A group is usually a
+ #: subtree, so this is normally that subtree's directories.
+ dirs: list[str]
+ target_path: str
+ dominant_layer: str = ""
+ #: Set when the group is a single directory larger than the ceiling and
+ #: therefore could not be split by any path-based rule.
+ oversized: bool = False
+ structural_key: str = field(default="", repr=False)
+
+ def __post_init__(self) -> None:
+ if not self.structural_key:
+ self.structural_key = member_structural_key(
+ self.members, prefix=STRUCTURAL_KEY_PREFIX
+ )
+
+ @property
+ def file_count(self) -> int:
+ return len(self.members)
+
+
+# ---------------------------------------------------------------------------
+# Directory tree
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class _Node:
+ path: str
+ own: list[str] = field(default_factory=list)
+ children: dict[str, _Node] = field(default_factory=dict)
+ subtree: int = 0
+
+ def child(self, name: str) -> _Node:
+ node = self.children.get(name)
+ if node is None:
+ node = _Node(path=f"{self.path}/{name}" if self.path else name)
+ self.children[name] = node
+ return node
+
+
+def _build_tree(paths: Iterable[str]) -> _Node:
+ root = _Node(path="")
+ for path in paths:
+ parts = path.split("/")
+ node = root
+ for part in parts[:-1]:
+ node = node.child(part)
+ node.own.append(path)
+ _count(root)
+ return root
+
+
+def _count(node: _Node) -> int:
+ node.subtree = len(node.own) + sum(_count(c) for c in node.children.values())
+ return node.subtree
+
+
+def _subtree_files(node: _Node) -> list[str]:
+ out = list(node.own)
+ for _name, child in sorted(node.children.items()):
+ out.extend(_subtree_files(child))
+ return out
+
+
+def _dirs_of(paths: Sequence[str]) -> list[str]:
+ return sorted({p.rsplit("/", 1)[0] if "/" in p else "" for p in paths})
+
+
+# ---------------------------------------------------------------------------
+# Partition
+# ---------------------------------------------------------------------------
+
+
+def _dominant(paths: Sequence[str], layer_of_file: dict[str, str]) -> str:
+ """The layer most of *paths* belong to, ties broken on the layer id.
+
+ Matches ``page_tree._dominant_layer`` deliberately: a group placed under a
+ layer in the tree and a group whose merge decision consulted a different
+ notion of "its layer" would disagree about where it belongs.
+ """
+ counts: dict[str, int] = {}
+ for path in paths:
+ layer = layer_of_file.get(path)
+ if layer:
+ counts[layer] = counts.get(layer, 0) + 1
+ if not counts:
+ return ""
+ return sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0][0]
+
+
+def _target_path(members: Sequence[str], dirs: Sequence[str]) -> str:
+ """The directory that names this group.
+
+ The shallowest directory containing every member, which for a subtree group
+ is that subtree's root and for a run of siblings is their shared parent.
+ Derived from the members rather than picked from a list, so it does not
+ depend on which directory happened to sort first.
+
+ **This value is the page id** (``compute_page_id`` is
+ ``"{page_type}:{target_path}"``), so two groups sharing one would be two
+ pages sharing a row. The partition splits a directory's own files from its
+ subdirectories' files, so a "files directly in ``routers``" group and a
+ "everything under ``routers/``" group both compute ``routers`` here.
+ :func:`_assign_targets` resolves that; this function is the unqualified
+ answer.
+ """
+ if not members:
+ return dirs[0] if dirs else ""
+ split = [m.split("/")[:-1] for m in members]
+ common = split[0]
+ for parts in split[1:]:
+ limit = min(len(common), len(parts))
+ cut = limit
+ for i in range(limit):
+ if common[i] != parts[i]:
+ cut = i
+ break
+ common = common[:cut]
+ return "/".join(common)
+
+
+class _Partitioner:
+ def __init__(self, params: GroupingParams, layer_of_file: dict[str, str]):
+ self.params = params
+ self.layer_of_file = layer_of_file
+ self.groups: list[ConceptGroup] = []
+
+ def make(self, members: Sequence[str]) -> ConceptGroup:
+ ordered = sorted(members)
+ dirs = _dirs_of(ordered)
+ # A group over the ceiling whose files all sit directly in one
+ # directory has hit the floor of what a path-based rule can do: there
+ # is no sub-path to split on. Chunking such a directory by filename
+ # order would split it, but the chunk boundaries would move whenever a
+ # file was added, and every chunk's identity hashes its members — so a
+ # one-file commit would remint every page in the directory. Stability
+ # is a correctness requirement here and page size is a quality target,
+ # so the group stays whole and says that it is oversized.
+ oversized = len(ordered) > self.params.max_files and len(dirs) == 1
+ return ConceptGroup(
+ members=ordered,
+ dirs=dirs,
+ target_path=_target_path(ordered, dirs),
+ dominant_layer=_dominant(ordered, self.layer_of_file),
+ oversized=oversized,
+ )
+
+ def _same_layer(self, left: Sequence[str], right: Sequence[str]) -> bool:
+ """Whether two pending runs may be merged across a layer boundary.
+
+ A merge that straddles two architectural layers produces a page about
+ no single thing, which is the failure the layer hint exists to prevent.
+ The exception is a run too thin to stand alone: a three-file group with
+ a title is worse documentation than a slightly impure eight-file one.
+ """
+ if len(left) + len(right) < self.params.min_files:
+ return True
+ a = _dominant(left, self.layer_of_file)
+ b = _dominant(right, self.layer_of_file)
+ return not a or not b or a == b
+
+ def visit(self, node: _Node) -> list[str]:
+ """Emit groups for *node*'s subtree; return the files left for its parent.
+
+ The returned files are ones that did not amount to a group on their
+ own. The parent either merges them with an adjacent sibling's leftovers
+ or, at the top, flushes them as their own group.
+ """
+ if node.subtree <= self.params.max_files:
+ return _subtree_files(node)
+
+ pending: list[str] = list(node.own)
+ for _name, child in sorted(node.children.items()):
+ left = self.visit(child)
+ if not left:
+ continue
+ if len(pending) + len(left) <= self.params.max_files and self._same_layer(
+ pending, left
+ ):
+ pending.extend(left)
+ continue
+ if pending:
+ self.groups.append(self.make(pending))
+ # A child only returns files when its whole subtree fits under the
+ # ceiling, so this can never itself overflow. An over-ceiling
+ # single directory is emitted by the recursion below, not here.
+ pending = list(left)
+
+ # A thin remainder is folded back into the group before it rather than
+ # left as a two-file page, but only when they share a layer and the
+ # result still fits.
+ if pending and len(pending) < self.params.min_files and self.groups:
+ last = self.groups[-1]
+ if len(last.members) + len(pending) <= self.params.max_files and self._same_layer(
+ last.members, pending
+ ):
+ merged = self.make(last.members + pending)
+ self.groups[-1] = merged
+ pending = []
+ if pending:
+ self.groups.append(self.make(pending))
+ return []
+
+
+def _absorb_thin(
+ groups: list[ConceptGroup], part: _Partitioner
+) -> list[ConceptGroup]:
+ """Fold groups too small to deserve a page into their closest neighbour.
+
+ The recursion flushes a remainder wherever a subtree boundary falls, so a
+ directory holding two files next to one holding twenty-nine can leave a
+ two-file group standing on its own. A page about two files is not a
+ concept, it is a directory listing with a title.
+
+ Neighbours are considered in path order and the one sharing the longer
+ prefix wins, so a thin group joins the thing it is actually near rather
+ than whichever side happened to be checked first. A group with no legal
+ neighbour — the merge would breach the ceiling, or cross a layer — stays,
+ because a slightly thin page is better than a wrong one.
+ """
+ params = part.params
+ working = sorted(groups, key=lambda g: g.target_path)
+ changed = True
+ while changed:
+ changed = False
+ for i, group in enumerate(working):
+ if group.file_count >= params.min_files or len(working) == 1:
+ continue
+ best: int | None = None
+ best_rank = (-1, -1)
+ for j in (i - 1, i + 1):
+ if j < 0 or j >= len(working):
+ continue
+ other = working[j]
+ if other.file_count + group.file_count > params.max_files:
+ continue
+ # A same-layer neighbour is preferred, but a thin group takes
+ # a cross-layer neighbour over standing alone. The layer gate
+ # exists to stop a page spanning two subsystems; it should not
+ # produce a one-file page, which is a worse outcome by every
+ # measure the outline is judged on.
+ same = part._same_layer(other.members, group.members)
+ # Depth of the directory the two would share. ``"".split("/")``
+ # and ``"src".split("/")`` are both length one, so measuring by
+ # segment count rated "shares nothing" equal to "shares a
+ # top-level directory" and let the tie fall to whichever side
+ # was examined first.
+ common = _target_path(group.members + other.members, [])
+ shared = 0 if not common else common.count("/") + 1
+ rank = (1 if same else 0, shared)
+ if rank > best_rank:
+ best_rank = rank
+ best = j
+ if best is None:
+ continue
+ merged = part.make(working[best].members + group.members)
+ working[best] = merged
+ del working[i]
+ changed = True
+ break
+ return working
+
+
+def _assign_targets(groups: list[ConceptGroup]) -> None:
+ """Give every group a distinct ``target_path`` that is still a real directory.
+
+ A group whose shallowest common directory also physically holds some of its
+ files keeps that directory: it is the one that owns it. A group covering
+ only subdirectories of that directory takes its own first directory
+ instead, which is real, is inside the group, and cannot be claimed by
+ anyone else because the partition never splits one directory's files across
+ two groups.
+
+ That argument is sound but it is an argument, and the cost of it being
+ wrong is two pages silently sharing a row, so the loop below checks the
+ result and walks to the next candidate rather than trusting it.
+ """
+ taken: set[str] = set()
+ # Largest first so the group with the strongest claim to a directory keeps
+ # the shortest name; ties break on the key so the order never depends on
+ # the traversal.
+ for group in sorted(groups, key=lambda g: (-g.file_count, g.structural_key)):
+ owned = _target_path(group.members, group.dirs)
+ candidates = [owned] if owned in group.dirs else []
+ candidates.extend(d for d in group.dirs if d and d not in candidates)
+ if owned and owned not in candidates:
+ candidates.append(owned)
+ # ``None`` rather than "" for "nothing chosen yet": the repository root
+ # is a real, legitimate directory whose path *is* the empty string, and
+ # a falsy check cannot tell it apart from failure. It could not, once —
+ # a group holding root-level files (``setup.py``, ``manage.py``,
+ # ``index.js``) fell through to the hash fallback and was titled after
+ # its own digest.
+ chosen: str | None = None
+ for candidate in candidates:
+ if candidate not in taken:
+ chosen = candidate
+ break
+ if chosen is None:
+ # Every real directory is spoken for. Fall back to the members'
+ # common prefix qualified by the group's identity, which is stable
+ # and unique even though it is no longer only a directory.
+ chosen = f"{owned}#{group.structural_key.rsplit('-', 1)[-1]}"
+ group.target_path = chosen
+ taken.add(chosen)
+
+
+def group_files(
+ paths: Iterable[str],
+ *,
+ layer_of_file: dict[str, str] | None = None,
+ params: GroupingParams | None = None,
+) -> list[ConceptGroup]:
+ """Partition *paths* into bounded, path-local concept groups.
+
+ *paths* must already exclude test files (D8) — this function does not
+ filter, so a caller that wants tests in gets them in and the count reflects
+ it. *layer_of_file* maps a file to its curated layer id and only affects
+ whether two adjacent runs may be merged; grouping works without it.
+
+ Groups come back in path order, which is also the order their members were
+ walked, so the sequence itself is stable.
+ """
+ files = sorted({p.replace("\\", "/") for p in paths if p})
+ if not files:
+ return []
+ tree = _build_tree(files)
+ resolved = params or params_for(len(files))
+ part = _Partitioner(resolved, layer_of_file or {})
+ leftover = part.visit(tree)
+ if leftover:
+ # The whole repository fits under the ceiling: one group, which is the
+ # honest answer for a tiny repository.
+ part.groups.append(part.make(leftover))
+ final = _absorb_thin(part.groups, part)
+ _assign_targets(final)
+ final.sort(key=lambda g: (g.target_path, g.structural_key))
+ return final
diff --git a/packages/core/src/repowise/core/generation/concept_tree/models.py b/packages/core/src/repowise/core/generation/concept_tree/models.py
new file mode 100644
index 000000000..aa2b0e61e
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/concept_tree/models.py
@@ -0,0 +1,164 @@
+"""The outline itself, and the numbers that say whether it is any good.
+
+An outline is a two-level thing — sections that group pages — but the tree it
+becomes is not, because sections are not pages. A section exists to be a
+heading; giving it a row would mean inventing pages nobody asked for and would
+break the property that placement adds no pages. So a section is carried here
+as a label plus a number, and the pages under it inherit the number.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .grouping import ConceptGroup
+
+
+@dataclass
+class ConceptPage:
+ """One leaf of the outline: a named group of files."""
+
+ title: str
+ #: One sentence saying what the page covers and what it deliberately does
+ #: not. The "and does not" half is the part that stops adjacent pages
+ #: describing each other.
+ scope: str
+ group: ConceptGroup
+ #: Dotted number within the outline, e.g. ``"3.2"``. Assigned by the
+ #: planner after ordering, never by the model.
+ number: str = ""
+ #: False when the title came from the deterministic fallback because the
+ #: model skipped this group. Long inputs get skipped near the tail, so
+ #: these are what the repair pass goes back for.
+ named_by_model: bool = False
+
+ @property
+ def target_path(self) -> str:
+ return self.group.target_path
+
+ @property
+ def structural_key(self) -> str:
+ return self.group.structural_key
+
+ @property
+ def members(self) -> list[str]:
+ return self.group.members
+
+
+@dataclass
+class ConceptSection:
+ """A heading over one or more pages."""
+
+ title: str
+ pages: list[ConceptPage] = field(default_factory=list)
+ number: str = ""
+
+
+@dataclass
+class ConceptOutline:
+ """A complete, ordered outline over a repository's production files."""
+
+ sections: list[ConceptSection] = field(default_factory=list)
+ #: Terms lifted from the repository's own docs and bound to a real group.
+ #: Present for provenance: a title that came from the project's vocabulary
+ #: should be traceable to the sentence that supplied it.
+ vocabulary: dict[str, str] = field(default_factory=dict)
+ #: ``"llm"`` when a model named the groups, ``"deterministic"`` when the
+ #: keyless fallback did. One axis, per D5: the map is free, the prose
+ #: needs a key.
+ naming_mode: str = "deterministic"
+
+ @property
+ def pages(self) -> list[ConceptPage]:
+ return [p for s in self.sections for p in s.pages]
+
+ def number_sections(self) -> None:
+ """Assign dotted numbers by position. Sections are 1-based."""
+ for si, section in enumerate(self.sections, start=1):
+ section.number = str(si)
+ for pi, page in enumerate(section.pages, start=1):
+ page.number = f"{si}.{pi}"
+
+
+@dataclass
+class OutlineReport:
+ """What the validator measured.
+
+ Deliberately numbers rather than a boolean. "The outline validated" says
+ nothing a reader can act on; "two directories unclaimed, nine paths
+ invented" says exactly where the planner went wrong, and the probes showed
+ those two failures move independently.
+ """
+
+ total_files: int = 0
+ covered_files: int = 0
+ unclaimed_files: list[str] = field(default_factory=list)
+ double_claimed_files: list[str] = field(default_factory=list)
+ #: Paths the model emitted that are not in the index. The probes measured
+ #: one on a good run and nine on a constrained one, so this fires.
+ invented_paths: list[str] = field(default_factory=list)
+ page_count: int = 0
+ section_count: int = 0
+ empty_pages: list[str] = field(default_factory=list)
+ oversized_pages: list[str] = field(default_factory=list)
+ single_child_sections: list[str] = field(default_factory=list)
+ duplicate_titles: list[str] = field(default_factory=list)
+ bad_length_titles: list[str] = field(default_factory=list)
+ bare_directory_titles: list[str] = field(default_factory=list)
+ duplicate_structural_keys: list[str] = field(default_factory=list)
+ #: Two pages resolving to one ``target_path``. That is two pages sharing a
+ #: page id, so one silently overwrites the other on persist.
+ duplicate_target_paths: list[str] = field(default_factory=list)
+ test_paths_included: list[str] = field(default_factory=list)
+ title_word_avg: float = 0.0
+
+ @property
+ def coverage(self) -> float:
+ return self.covered_files / self.total_files if self.total_files else 0.0
+
+ @property
+ def hard_failures(self) -> list[str]:
+ """The failures that make an outline wrong rather than untidy.
+
+ Coverage and fabrication are correctness; a title one word too long is
+ taste. Only the first kind blocks.
+ """
+ out: list[str] = []
+ if self.unclaimed_files:
+ out.append(f"{len(self.unclaimed_files)} unclaimed files")
+ if self.double_claimed_files:
+ out.append(f"{len(self.double_claimed_files)} double-claimed files")
+ if self.invented_paths:
+ out.append(f"{len(self.invented_paths)} invented paths")
+ if self.empty_pages:
+ out.append(f"{len(self.empty_pages)} pages claiming no files")
+ if self.duplicate_structural_keys:
+ out.append(f"{len(self.duplicate_structural_keys)} duplicate structural keys")
+ if self.duplicate_target_paths:
+ out.append(f"{len(self.duplicate_target_paths)} duplicate target paths")
+ if self.test_paths_included:
+ out.append(f"{len(self.test_paths_included)} test files in the tree")
+ return out
+
+ @property
+ def ok(self) -> bool:
+ return not self.hard_failures
+
+ def as_lines(self) -> list[str]:
+ """The report as the generation log and the progress doc want it."""
+ return [
+ f"coverage {self.covered_files}/{self.total_files} = "
+ f"{100 * self.coverage:.1f}%",
+ f"pages {self.page_count} in {self.section_count} sections",
+ f"unclaimed {len(self.unclaimed_files)}",
+ f"double-claimed {len(self.double_claimed_files)}",
+ f"invented paths {len(self.invented_paths)}",
+ f"empty pages {len(self.empty_pages)}",
+ f"oversized {len(self.oversized_pages)}",
+ f"one-child sects {len(self.single_child_sections)}",
+ f"dup titles {len(self.duplicate_titles)}",
+ f"title words avg {self.title_word_avg:.1f}",
+ f"dup struct keys {len(self.duplicate_structural_keys)}",
+ f"dup target paths {len(self.duplicate_target_paths)}",
+ f"test files in {len(self.test_paths_included)}",
+ ]
diff --git a/packages/core/src/repowise/core/generation/concept_tree/naming.py b/packages/core/src/repowise/core/generation/concept_tree/naming.py
new file mode 100644
index 000000000..f5a4c8771
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/concept_tree/naming.py
@@ -0,0 +1,428 @@
+"""Name and order the groups. The model never decides which files they hold.
+
+This is the half of the planner an LLM is good at, and it is bounded so that
+the half it is bad at cannot leak back in. Groups arrive with opaque ids and
+the model returns a title, a scope sentence and a section for each id. It is
+never shown a blank canvas and never asked which directory belongs where, so
+the two failures the probes measured — dropping directories and inventing
+plausible-looking paths — have nowhere to enter through. A group the model
+forgets to name keeps a deterministic name; a group id it invents is discarded.
+Coverage is therefore unchanged by anything the model does or fails to do.
+
+The payload conveys breadth, never importance. Ranking the input by PageRank
+and sampling the top files produced 12.7% coverage against 91.7% for the full
+inventory, because hub files are infrastructure — ``models.py``, ``client.ts``,
+``__init__.py`` — and an outline built around them describes the plumbing
+rather than the product.
+
+Repo docs do not come in here as prose. Nine thousand characters of README
+injected alongside a 145-directory structural task collapsed the outline to
+three pages. Vocabulary arrives already bound to a group by
+:mod:`vocabulary`, as a short list of suggested names.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+
+import structlog
+
+from ..onboarding.grounding import check_grounding
+from .grouping import ConceptGroup
+
+logger = structlog.get_logger(__name__)
+
+MAX_TITLE_WORDS = 7
+MIN_TITLE_WORDS = 2
+
+_FENCE = re.compile(r"^```[a-zA-Z]*\n|\n```$")
+
+
+def parse_json_object(content: str) -> dict[str, Any]:
+ """Parse a JSON object out of a model response, tolerating decoration.
+
+ Providers here are asked for JSON by instruction rather than by an API
+ flag, because no provider in the set exposes a native structured-output
+ mode. So the response can arrive fenced, prefaced, or trailed with prose,
+ and the recovery ladder is the same one decision extraction already uses:
+ strip fences, parse, and failing that pull the outermost braces out.
+ Returns an empty dict rather than raising — an unparseable outline falls
+ back to deterministic names, which is a worse wiki, not a broken run.
+ """
+ text = (content or "").strip()
+ if text.startswith("```"):
+ text = "\n".join(
+ line for line in text.split("\n") if not line.strip().startswith("```")
+ )
+ try:
+ parsed = json.loads(text)
+ except (json.JSONDecodeError, ValueError):
+ match = re.search(r"\{.*\}", text, re.DOTALL)
+ if not match:
+ return {}
+ try:
+ parsed = json.loads(match.group())
+ except (json.JSONDecodeError, ValueError):
+ return {}
+ return parsed if isinstance(parsed, dict) else {}
+
+
+# ---------------------------------------------------------------------------
+# Deterministic naming — the keyless path (D5)
+# ---------------------------------------------------------------------------
+
+_STOP_SEGMENTS = {"src", "lib", "app", "packages", "core", "internal", "pkg", "source"}
+
+
+def _humanise(segment: str) -> str:
+ words = re.split(r"[-_.\s]+", segment)
+ out: list[str] = []
+ for word in words:
+ if not word:
+ continue
+ # Split camelCase so ``pageGenerator`` reads as two words.
+ parts = re.findall(r"[A-Z]+(?![a-z])|[A-Z][a-z]*|[a-z]+|\d+", word) or [word]
+ out.extend(p.capitalize() if p.islower() else p for p in parts)
+ return " ".join(out)
+
+
+def deterministic_title(group: ConceptGroup, layer_label: str = "") -> str:
+ """A readable name from structure alone, for a run with no provider.
+
+ Uses the last one or two meaningful path segments, prefixed by the layer
+ when that is what makes the name unambiguous. Not as good as a named
+ capability and not meant to be: per D5 the map is free and the prose needs
+ a key, so a keyless wiki gets a real tree with plainer labels rather than a
+ worse tree.
+ """
+ segments = [s for s in group.target_path.split("/") if s]
+ meaningful = [s for s in segments if s.lower() not in _STOP_SEGMENTS] or segments
+ tail = [_humanise(s) for s in meaningful[-2:]]
+ title = " ".join(tail).strip()
+ if layer_label and len(title.split()) < MIN_TITLE_WORDS:
+ title = f"{layer_label} {title}".strip()
+ if not title:
+ # The repository root has no path segments to name it by. It is a real
+ # place and a common one — top-level entry points live there — so it
+ # gets a name of its own rather than one derived from an empty path.
+ title = "Repository Root Files"
+ words = title.split()
+ if len(words) > MAX_TITLE_WORDS:
+ title = " ".join(words[:MAX_TITLE_WORDS])
+ if len(title.split()) < MIN_TITLE_WORDS:
+ title = f"{title} Components".strip()
+ return title
+
+
+def deterministic_scope(group: ConceptGroup) -> str:
+ """A scope line stating coverage and its boundary, with no model."""
+ # The root directory is the empty string, which is a real directory and
+ # not an absent one; filtering it out left a group reading "0 directories".
+ dirs = list(group.dirs)
+ if len(dirs) == 1:
+ where = dirs[0] or "the repository root"
+ else:
+ where = f"{len(dirs)} directories under {group.target_path or 'the repository root'}"
+ return (
+ f"Covers the {group.file_count} source files in {where}. "
+ "Does not cover code outside those directories, which is documented on "
+ "its own pages."
+ )
+
+
+# ---------------------------------------------------------------------------
+# LLM naming
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class NamedGroup:
+ """What the namer decided about one group."""
+
+ group: ConceptGroup
+ title: str
+ scope: str
+ section: str
+ order: int = 0
+ #: True when this name came from the fallback rather than the model, either
+ #: because there was no provider or because the model skipped the group.
+ fallback: bool = False
+
+
+SYSTEM_PROMPT = (
+ "You are an information architect naming the sections of a technical wiki "
+ "about a specific codebase, read by senior engineers and AI coding agents. "
+ "You name and order groups of files that have already been decided for you. "
+ "You never decide which files belong to which group. Reply with JSON only."
+)
+
+NAMING_INSTRUCTIONS = """\
+Below is the complete set of file groups for the `{repo}` codebase. The grouping \
+is fixed: every group is a contiguous part of the directory tree, every source \
+file is in exactly one group, and you must not change, merge, split or drop any \
+group.
+
+For EVERY group id, return a title, a one-sentence scope, and the section it \
+belongs to. Then order the sections.
+
+TITLES
+- Name the CAPABILITY or SUBSYSTEM, not the directory and not the layer. \
+Good: "Dependency Graph Construction", "Output Distillation", \
+"Code Health Scoring". Bad: "core/ingestion", "Ingestion Layer", \
+"Utilities and Helpers".
+- {min_words} to {max_words} words. Every title must be unique.
+- An enumerative title is right only when a group genuinely spans several small \
+areas, e.g. "Dead Code, Security, and Change Risk".
+- Where a suggested name is given for a group, prefer it: it is the project's \
+own word for this, taken from its documentation.
+
+SCOPE
+- One sentence saying what the page covers AND what it deliberately does not, \
+so that adjacent pages do not describe each other.
+- Refer only to paths that appear in that group's own listing. Do not name a \
+file or directory that is not in the input.
+
+SECTIONS
+- Group the pages into {min_sections} to {max_sections} sections. Order them as \
+a narrative: orientation first, then the core engine, then interfaces in order \
+of distance from the engine, then operations, then reference.
+- No section may contain exactly one page.
+- Section titles are 1 to 5 words.
+
+Return ONLY this JSON:
+{{"sections": [{{"title": "...", "groups": ["g01", "g07", ...]}}],
+ "names": {{"g01": {{"title": "...", "scope": "..."}}, ...}}}}
+
+GROUPS:
+"""
+
+
+def _sample_names(members: list[str], limit: int) -> list[str]:
+ """Filenames spread across the group's directories, not the first *limit*.
+
+ Taking the alphabetically-first names from a group spanning two
+ directories draws all of them from whichever directory sorts first, so the
+ namer sees half the page and names it after that half. Round-robin gives
+ every directory a turn before any gets a second.
+ """
+ by_dir: dict[str, list[str]] = {}
+ for member in sorted(members):
+ by_dir.setdefault(member.rsplit("/", 1)[0], []).append(
+ member.rsplit("/", 1)[-1]
+ )
+ out: list[str] = []
+ order = sorted(by_dir)
+ depth = 0
+ while len(out) < limit:
+ added = False
+ for directory in order:
+ names = by_dir[directory]
+ if depth < len(names):
+ out.append(names[depth])
+ added = True
+ if len(out) >= limit:
+ break
+ if not added:
+ break
+ depth += 1
+ return out
+
+
+def _common_prefix(dirs: list[str]) -> str:
+ """The deepest directory every entry of *dirs* sits under."""
+ if not dirs:
+ return ""
+ split = [d.split("/") for d in dirs]
+ common = split[0]
+ for parts in split[1:]:
+ cut = min(len(common), len(parts))
+ for i in range(cut):
+ if common[i] != parts[i]:
+ cut = i
+ break
+ common = common[:cut]
+ return "/".join(common)
+
+
+def build_payload(
+ groups: list[ConceptGroup],
+ *,
+ layer_labels: dict[str, str] | None = None,
+ summaries: dict[str, str] | None = None,
+ suggestions: dict[str, str] | None = None,
+ entry_points: set[str] | None = None,
+ max_filenames: int = 6,
+) -> tuple[dict[str, Any], dict[str, ConceptGroup]]:
+ """Build the namer's payload and the id-to-group index that decodes it.
+
+ Ids are positional (``g01``) rather than derived from the path, so the
+ model has nothing path-shaped to echo back and a fabricated id is
+ immediately recognisable as one. Filenames are included because they carry
+ what a directory name does not, but they are basenames only: a model given
+ full paths will quote them, and a quoted path is a citation we would then
+ have to verify.
+ """
+ labels = layer_labels or {}
+ mod_summaries = summaries or {}
+ hints = suggestions or {}
+ entries = entry_points or set()
+
+ index: dict[str, ConceptGroup] = {}
+ entries_out: list[dict[str, Any]] = []
+ for i, group in enumerate(groups, start=1):
+ gid = f"g{i:02d}"
+ index[gid] = group
+ entry: dict[str, Any] = {
+ "id": gid,
+ "dir": group.target_path,
+ "files": group.file_count,
+ "names": _sample_names(group.members, max_filenames),
+ }
+ if len(group.dirs) > 1:
+ # Relative to what the directories share, not to ``target_path``.
+ # The target is one of the group's directories and is not always a
+ # prefix of the others: a page covering ``ingestion/git_indexer``
+ # and ``ingestion/graph`` is targeted at the first, and stripping
+ # that prefix from the second yielded an empty string. Both
+ # directories then arrived as ".", so the namer could not see that
+ # the page covered two subsystems and named it after one.
+ common = _common_prefix(group.dirs)
+ cut = len(common) + 1 if common else 0
+ entry["subdirs"] = [d[cut:] or d.rsplit("/", 1)[-1] for d in group.dirs[:8]]
+ label = labels.get(group.dominant_layer)
+ if label:
+ entry["layer"] = label
+ summary = mod_summaries.get(group.target_path)
+ if summary:
+ entry["summary"] = summary.replace("\n", " ")[:200]
+ hint = hints.get(gid)
+ if hint:
+ entry["suggested_name"] = hint
+ group_entries = sorted(e.rsplit("/", 1)[-1] for e in entries if e in set(group.members))
+ if group_entries:
+ entry["entry_points"] = group_entries[:3]
+ entries_out.append(entry)
+
+ return {"repo": "", "groups": entries_out}, index
+
+
+_MAX_TITLE_CHARS = 120
+
+
+def _clean_title(raw: Any) -> str:
+ # Only a string is a title. A dict or a list stringifies into something
+ # that looks like a title to every downstream check and reads as garbage
+ # to a person.
+ if not isinstance(raw, str):
+ return ""
+ title = raw.strip().strip('"').strip()
+ title = re.sub(r"\s+", " ", title)
+ # A model asked for a title sometimes returns "Section 3: Ingestion".
+ title = re.sub(r"^\d+(\.\d+)*[.:)\-]\s*", "", title)
+ # Length is bounded here rather than left to the validator, which treats
+ # it as taste. An unbounded title is a rendering problem, not a taste one.
+ if len(title) > _MAX_TITLE_CHARS:
+ title = title[:_MAX_TITLE_CHARS].rsplit(" ", 1)[0]
+ return title
+
+
+def decode_response(
+ data: dict[str, Any],
+ index: dict[str, ConceptGroup],
+ *,
+ layer_labels: dict[str, str] | None = None,
+) -> tuple[list[NamedGroup], list[str]]:
+ """Turn the model's JSON into named groups, keeping the grouping intact.
+
+ Returns ``(named, invented_ids)``. Every group in *index* comes back
+ exactly once whatever the model said: an id it skipped gets a
+ deterministic name, an id it invented is dropped and reported. This is the
+ property that makes coverage independent of the model.
+ """
+ labels = layer_labels or {}
+ names = data.get("names")
+ names = names if isinstance(names, dict) else {}
+ sections = data.get("sections")
+ sections = sections if isinstance(sections, list) else []
+
+ section_of: dict[str, str] = {}
+ order_of: dict[str, int] = {}
+ section_rank: dict[str, int] = {}
+ invented: list[str] = []
+ counter = 0
+ for rank, section in enumerate(sections):
+ if not isinstance(section, dict):
+ continue
+ title = _clean_title(section.get("title"))
+ if not title:
+ continue
+ section_rank.setdefault(title, rank)
+ # A scalar here is not hypothetical: a model asked for a list of ids
+ # returned a count instead, and ``or []`` passes a non-empty scalar
+ # straight through to the loop. The whole point of this function is
+ # that no model response can break the run, so the shape is checked
+ # rather than assumed.
+ raw_groups = section.get("groups")
+ if not isinstance(raw_groups, (list, tuple)):
+ raw_groups = []
+ for gid in raw_groups:
+ if not isinstance(gid, (str, int)):
+ continue
+ gid = str(gid)
+ if gid not in index:
+ invented.append(gid)
+ continue
+ if gid in section_of:
+ # Claimed twice. The first section wins, deterministically.
+ continue
+ section_of[gid] = title
+ order_of[gid] = counter
+ counter += 1
+
+ named: list[NamedGroup] = []
+ for gid, group in index.items():
+ entry = names.get(gid)
+ entry = entry if isinstance(entry, dict) else {}
+ title = _clean_title(entry.get("title"))
+ scope = str(entry.get("scope") or "").strip()
+ fallback = False
+ if not title:
+ title = deterministic_title(group, labels.get(group.dominant_layer, ""))
+ fallback = True
+ if not scope:
+ scope = deterministic_scope(group)
+ section = section_of.get(gid) or labels.get(group.dominant_layer) or "Reference"
+ named.append(
+ NamedGroup(
+ group=group,
+ title=title,
+ scope=scope,
+ section=section,
+ order=order_of.get(gid, len(index) + len(named)),
+ fallback=fallback,
+ )
+ )
+
+ named.sort(key=lambda n: (section_rank.get(n.section, len(sections)), n.order,
+ n.group.target_path))
+ return named, sorted(set(invented))
+
+
+def ground_scopes(named: list[NamedGroup]) -> list[str]:
+ """Strip citations from scope sentences that the group's files do not support.
+
+ Reuses the onboarding grounding check rather than repeating its shape
+ rules: a backticked token that looks like a path or a symbol and is absent
+ from the group's own file list loses its backticks, so it reads as prose
+ instead of as a verified reference. Returns every token demoted, which the
+ validator counts as an invented path.
+ """
+ ungrounded: list[str] = []
+ for entry in named:
+ cleaned, bad = check_grounding(entry.scope, entry.group.members)
+ entry.scope = cleaned
+ ungrounded.extend(bad)
+ return ungrounded
diff --git a/packages/core/src/repowise/core/generation/concept_tree/planner.py b/packages/core/src/repowise/core/generation/concept_tree/planner.py
new file mode 100644
index 000000000..73e29e3ee
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/concept_tree/planner.py
@@ -0,0 +1,474 @@
+"""Plan the outline: group, name, validate, repair, validate again.
+
+Two passes rather than one, because a single call cannot hold coverage and
+allocation at the same time — pushing on structure made a planner drop
+directories and invent paths at nine times the rate. Here the two jobs are
+separated at the source, so the second pass has almost nothing to do: grouping
+already guarantees coverage, and repair only ever revisits names.
+
+The repair pass sees only what failed. Handing back the whole tree invites the
+model to rewrite the parts that were fine, which is how a "fix these four
+titles" request turns into a different outline.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from .grouping import ConceptGroup, GroupingParams, group_files, params_for
+from .models import ConceptOutline, ConceptPage, ConceptSection, OutlineReport
+from .naming import (
+ NAMING_INSTRUCTIONS,
+ SYSTEM_PROMPT,
+ NamedGroup,
+ _humanise,
+ build_payload,
+ decode_response,
+ deterministic_scope,
+ deterministic_title,
+ ground_scopes,
+ parse_json_object,
+)
+from .validation import validate_outline
+from .vocabulary import bind_terms, extract_terms
+
+logger = structlog.get_logger(__name__)
+
+COST_OPERATION = "outline_planning"
+
+MIN_SECTIONS = 5
+MAX_SECTIONS = 11
+
+
+@dataclass
+class PlannerInputs:
+ """Everything the planner needs, already resolved by the caller.
+
+ Deliberately plain data. The planner is then testable without a pipeline,
+ a database or a provider, which matters because its interesting properties
+ are about determinism and those need to be asserted cheaply and often.
+ """
+
+ repo_name: str
+ #: Production source files, tests already excluded (D8).
+ production_files: list[str]
+ repo_root: Path | None = None
+ #: File path -> curated layer id.
+ layer_of_file: dict[str, str] = field(default_factory=dict)
+ #: Layer id -> display name, for section fallbacks and payload hints.
+ layer_labels: dict[str, str] = field(default_factory=dict)
+ #: Directory -> module summary written from the code.
+ summaries: dict[str, str] = field(default_factory=dict)
+ entry_points: set[str] = field(default_factory=set)
+ #: Test files, kept only so the validator can prove none leaked in.
+ test_files: set[str] = field(default_factory=set)
+
+
+def _billed_as(provider: Any, operation: str) -> Iterator[None]:
+ """Label this step's spend on the Costs page; no-op without a tracker."""
+ tracker = getattr(provider, "_cost_tracker", None)
+ if tracker is not None and hasattr(tracker, "record_as"):
+ with tracker.record_as(operation):
+ yield
+ else:
+ yield
+
+
+def _build_outline(named: list[NamedGroup]) -> ConceptOutline:
+ """Assemble sections from named groups, preserving the namer's order."""
+ outline = ConceptOutline()
+ by_section: dict[str, ConceptSection] = {}
+ for entry in named:
+ section = by_section.get(entry.section)
+ if section is None:
+ section = ConceptSection(title=entry.section)
+ by_section[entry.section] = section
+ outline.sections.append(section)
+ section.pages.append(
+ ConceptPage(
+ title=entry.title,
+ scope=entry.scope,
+ group=entry.group,
+ named_by_model=not entry.fallback,
+ )
+ )
+ outline.number_sections()
+ return outline
+
+
+def _merge_thin_sections(outline: ConceptOutline) -> None:
+ """Fold a section holding a single page into its neighbour.
+
+ A heading over one page is not a grouping, it is a heading pretending to
+ be one, and it makes a table of contents look padded. The page keeps its
+ own title, so nothing is lost by moving it.
+ """
+ if len(outline.sections) <= 1:
+ return
+ changed = True
+ while changed and len(outline.sections) > 1:
+ changed = False
+ for i, section in enumerate(outline.sections):
+ if len(section.pages) != 1:
+ continue
+ target = i - 1 if i > 0 else i + 1
+ outline.sections[target].pages.extend(section.pages)
+ del outline.sections[i]
+ changed = True
+ break
+ outline.number_sections()
+
+
+def _disambiguate_titles(named: list[NamedGroup]) -> None:
+ """Make path-derived titles unique by qualifying them with their parent.
+
+ Two groups carved out of one directory tree — one holding a directory's own
+ files, one holding its subdirectories — produce the same name from the same
+ path segments. Rather than numbering them, which tells a reader nothing,
+ the later one borrows a segment from further up its own path.
+ """
+ seen: dict[str, int] = {}
+ for entry in sorted(named, key=lambda n: (n.title.lower(), n.group.target_path)):
+ key = entry.title.lower()
+ if key not in seen:
+ seen[key] = 1
+ continue
+ segments = [s for s in entry.group.target_path.split("/") if s]
+ for extra in reversed(segments[:-1]):
+ candidate = f"{deterministic_title(entry.group)} {_humanise(extra)}".strip()
+ candidate = " ".join(dict.fromkeys(candidate.split()))
+ if candidate.lower() not in seen and len(candidate.split()) <= 7:
+ entry.title = candidate
+ break
+ else:
+ seen[key] += 1
+ entry.title = f"{entry.title} {seen[key]}"
+ seen[entry.title.lower()] = 1
+
+
+def plan_deterministic(
+ inputs: PlannerInputs, *, params: GroupingParams | None = None
+) -> tuple[ConceptOutline, list[ConceptGroup]]:
+ """The keyless outline: real structure, names derived from paths and layers.
+
+ Per D5 this is the same tree the LLM path produces — identical grouping,
+ identical identities — with plainer titles. Adding a key upgrades the
+ prose, never the shape, so a user who adds one later does not get their
+ wiki re-partitioned underneath them.
+
+ That guarantee only holds if both paths partition with the same bounds, so
+ *params* is threaded rather than re-derived. It was not, once: the caller's
+ bounds were dropped here and the two paths produced different page counts
+ for the same repository.
+ """
+ groups = group_files(
+ inputs.production_files, layer_of_file=inputs.layer_of_file, params=params
+ )
+ named = [
+ NamedGroup(
+ group=g,
+ title=deterministic_title(g, inputs.layer_labels.get(g.dominant_layer, "")),
+ scope=deterministic_scope(g),
+ section=inputs.layer_labels.get(g.dominant_layer) or "Reference",
+ order=i,
+ fallback=True,
+ )
+ for i, g in enumerate(groups)
+ ]
+ _disambiguate_titles(named)
+ outline = _build_outline(named)
+ _merge_thin_sections(outline)
+ outline.naming_mode = "deterministic"
+ return outline, groups
+
+
+_REPAIR_INSTRUCTIONS = """\
+These page titles from the outline you just produced need fixing. Nothing else \
+about the outline is changing — do not restructure it, and do not rename any \
+page that is not listed here.
+
+Return a corrected title AND a scope sentence for exactly these group ids.
+
+The title is {min_words} to {max_words} words, names the capability rather than \
+the directory or the layer, and is unique across the whole outline.
+
+The scope is a full English sentence saying what the page covers and what it \
+deliberately does not. It is never a path: do not echo the `dir` value back.
+
+Titles already in use elsewhere in the outline (do not reuse any of these):
+{taken}
+
+GROUPS TO RENAME:
+{failures}
+
+Return ONLY this JSON:
+{{"names": {{"g07": {{"title": "...", "scope": "..."}}, ...}}}}
+"""
+
+
+def _ground_pages(outline: ConceptOutline) -> list[str]:
+ """Strip unsupported citations from every page's scope. Returns the tokens."""
+ from ..onboarding.grounding import check_grounding
+
+ ungrounded: list[str] = []
+ for page in outline.pages:
+ cleaned, bad = check_grounding(page.scope, page.members)
+ page.scope = cleaned
+ ungrounded.extend(bad)
+ return ungrounded
+
+
+def _usable_scope(scope: str, target_path: str) -> bool:
+ """Whether a returned scope is a sentence rather than an echoed path.
+
+ Asked for a title and a scope for a group described by its directory, a
+ model will sometimes hand the directory back in the scope field. That
+ reads as a broken page rather than a terse one, so it is rejected and the
+ deterministic sentence stands.
+ """
+ text = scope.strip()
+ if len(text.split()) < 4:
+ return False
+ if text.rstrip("/") == target_path.rstrip("/"):
+ return False
+ # A lone path has no spaces before its first separator run; a sentence does.
+ return "/" not in text.split(" ", 1)[0] or " " in text.strip()
+
+
+def _repair_targets(outline: ConceptOutline, report: OutlineReport) -> set[str]:
+ """The page titles worth a second call. Everything else is left alone.
+
+ Three kinds qualify: titles the model repeated, titles that are just the
+ directory respelled, and groups the model never named at all. The last is
+ the common one on a large repository — a request to name seventy groups
+ reliably loses a few near the end of the list — and it is also the
+ cheapest to fix, because the second call carries only the stragglers.
+ """
+ bad = set(report.duplicate_titles)
+ targets = {p.title for p in outline.pages if p.title.lower() in bad}
+ targets |= set(report.bare_directory_titles)
+ targets |= {t.rsplit(" (", 1)[0] for t in report.bad_length_titles}
+ targets |= {p.title for p in outline.pages if not p.named_by_model}
+ return targets
+
+
+async def plan_outline(
+ inputs: PlannerInputs,
+ *,
+ provider: Any | None = None,
+ deterministic: bool = False,
+ params: GroupingParams | None = None,
+ repair: bool = True,
+) -> tuple[ConceptOutline, OutlineReport]:
+ """Produce a validated outline for *inputs*.
+
+ Falls back to :func:`plan_deterministic` whenever there is no provider, the
+ run is in deterministic mode, or the model's response cannot be used. A
+ failed naming call costs the wiki its titles, never its structure.
+ """
+ all_files = set(inputs.production_files)
+ resolved = params or params_for(len(all_files))
+
+ if deterministic or provider is None:
+ outline, _groups = plan_deterministic(inputs, params=params)
+ report = validate_outline(
+ outline,
+ all_files=all_files,
+ test_files=inputs.test_files,
+ max_files_per_page=resolved.max_files,
+ )
+ return outline, report
+
+ groups = group_files(
+ inputs.production_files, layer_of_file=inputs.layer_of_file, params=params
+ )
+ payload, index = build_payload(
+ groups,
+ layer_labels=inputs.layer_labels,
+ summaries=inputs.summaries,
+ entry_points=inputs.entry_points,
+ )
+ payload["repo"] = inputs.repo_name
+
+ terms: list[str] = []
+ bound: dict[str, str] = {}
+ if inputs.repo_root is not None:
+ terms = extract_terms(inputs.repo_root)
+ bound = bind_terms(terms, index)
+ for gid, term in bound.items():
+ for entry in payload["groups"]:
+ if entry["id"] == gid:
+ entry["suggested_name"] = term
+ break
+ logger.info(
+ "concept_outline_vocabulary",
+ terms_found=len(terms),
+ terms_bound=len(bound),
+ )
+
+ sections_lo = min(MIN_SECTIONS, max(2, len(groups) // 8))
+ sections_hi = max(sections_lo + 1, min(MAX_SECTIONS, len(groups) // 3 or 2))
+ instructions = NAMING_INSTRUCTIONS.format(
+ repo=inputs.repo_name,
+ min_words=2,
+ max_words=7,
+ min_sections=sections_lo,
+ max_sections=sections_hi,
+ )
+ body = json.dumps(payload, separators=(",", ":"))
+
+ data: dict[str, Any] = {}
+ try:
+ with _billed_as(provider, COST_OPERATION):
+ response = await provider.generate(
+ system_prompt=SYSTEM_PROMPT,
+ user_prompt=instructions + body,
+ max_tokens=16000,
+ temperature=0.2,
+ )
+ data = parse_json_object(getattr(response, "content", "") or "")
+ except Exception as exc:
+ logger.warning("concept_outline_naming_failed", error=str(exc))
+
+ # Decoding sits inside the guard too. It used to sit outside, which meant a
+ # response that parsed as JSON but held the wrong shape reached the decoder
+ # and could take the run down — the one failure mode this design exists to
+ # rule out. Decoding is defensive in its own right; this is the second line.
+ try:
+ named, invented_ids = decode_response(
+ data, index, layer_labels=inputs.layer_labels
+ )
+ except Exception as exc:
+ logger.warning("concept_outline_decode_failed", error=str(exc))
+ named, invented_ids = decode_response({}, index, layer_labels=inputs.layer_labels)
+ ungrounded = ground_scopes(named)
+ if invented_ids:
+ logger.warning("concept_outline_invented_group_ids", ids=invented_ids[:10])
+
+ outline = _build_outline(named)
+ _merge_thin_sections(outline)
+ outline.naming_mode = "llm" if any(not n.fallback for n in named) else "deterministic"
+ outline.vocabulary = {bound[gid]: index[gid].target_path for gid in bound}
+
+ report = validate_outline(
+ outline,
+ all_files=all_files,
+ test_files=inputs.test_files,
+ max_files_per_page=resolved.max_files,
+ )
+ report.invented_paths = sorted(set(report.invented_paths) | set(ungrounded))
+
+ if repair:
+ targets = _repair_targets(outline, report)
+ if targets:
+ await _repair_titles(outline, index, targets, provider=provider)
+ # Repair writes new prose, so it has to face the same citation
+ # check the first pass did. Grounding after the last write rather
+ # than after the first is the difference between checking the page
+ # and checking a draft of it.
+ ungrounded.extend(_ground_pages(outline))
+ report = validate_outline(
+ outline,
+ all_files=all_files,
+ test_files=inputs.test_files,
+ max_files_per_page=resolved.max_files,
+ )
+ report.invented_paths = sorted(set(report.invented_paths) | set(ungrounded))
+
+ logger.info(
+ "concept_outline_planned",
+ pages=report.page_count,
+ sections=report.section_count,
+ coverage=round(report.coverage, 4),
+ invented=len(report.invented_paths),
+ naming_mode=outline.naming_mode,
+ )
+ return outline, report
+
+
+async def _repair_titles(
+ outline: ConceptOutline,
+ index: dict[str, ConceptGroup],
+ targets: set[str],
+ *,
+ provider: Any,
+) -> None:
+ """Re-ask for just the failing titles, then apply only what improved.
+
+ A repair that made things worse is discarded: the replacement has to be
+ non-empty, correctly sized and not already in use, or the original stands.
+ That keeps a bad second call from being worse than no second call.
+ """
+ gid_of = {g.structural_key: gid for gid, g in index.items()}
+ failing = [p for p in outline.pages if p.title in targets]
+ if not failing:
+ return
+ taken = sorted({p.title for p in outline.pages if p.title not in targets})
+
+ lines = []
+ for page in failing:
+ gid = gid_of.get(page.structural_key, "")
+ names = sorted(m.rsplit("/", 1)[-1] for m in page.members)[:6]
+ lines.append(
+ json.dumps(
+ {
+ "id": gid,
+ "current_title": page.title,
+ "dir": page.target_path,
+ "files": len(page.members),
+ "names": names,
+ },
+ separators=(",", ":"),
+ )
+ )
+
+ prompt = _REPAIR_INSTRUCTIONS.format(
+ min_words=2,
+ max_words=7,
+ taken="\n".join(f"- {t}" for t in taken) or "(none)",
+ failures="\n".join(lines),
+ )
+ try:
+ with _billed_as(provider, COST_OPERATION):
+ response = await provider.generate(
+ system_prompt=SYSTEM_PROMPT,
+ user_prompt=prompt,
+ max_tokens=2000,
+ temperature=0.2,
+ )
+ data = parse_json_object(getattr(response, "content", "") or "")
+ except Exception as exc:
+ logger.warning("concept_outline_repair_failed", error=str(exc))
+ return
+
+ names = data.get("names")
+ if not isinstance(names, dict):
+ return
+ in_use = {p.title.lower() for p in outline.pages}
+ fixed = 0
+ for page in failing:
+ gid = gid_of.get(page.structural_key, "")
+ entry = names.get(gid)
+ if not isinstance(entry, dict):
+ continue
+ title = str(entry.get("title") or "").strip()
+ words = len(title.split())
+ if not title or not (2 <= words <= 7) or title.lower() in in_use:
+ continue
+ in_use.discard(page.title.lower())
+ in_use.add(title.lower())
+ page.title = title
+ page.named_by_model = True
+ scope = str(entry.get("scope") or "").strip()
+ if _usable_scope(scope, page.target_path):
+ page.scope = scope
+ fixed += 1
+ logger.info("concept_outline_repaired", requested=len(failing), applied=fixed)
diff --git a/packages/core/src/repowise/core/generation/concept_tree/validation.py b/packages/core/src/repowise/core/generation/concept_tree/validation.py
new file mode 100644
index 000000000..d95b50ae1
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/concept_tree/validation.py
@@ -0,0 +1,112 @@
+"""Check the outline against the index, and count what is wrong.
+
+Not a theoretical gap. Measured on this codebase, a planner run invented one
+directory and a more heavily constrained one invented nine, among them
+``core/generation/cost_tracker.py`` and ``web/src/lib/workspace-mode.ts`` —
+paths that read as if they belong here and do not exist. An unchecked citation
+like that reaches the reader as an authoritative reference, which is worse than
+saying nothing.
+
+Grouping is deterministic, so coverage and double-claiming ought to be true by
+construction. They are still asserted here, because "ought to be" is how the
+last two placement bugs got in, and because the repair pass can hand back a
+tree the model touched.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+
+from ..models import compute_page_id # noqa: F401 (documents the id contract)
+from .models import ConceptOutline, OutlineReport
+
+MIN_TITLE_WORDS = 2
+MAX_TITLE_WORDS = 7
+
+
+def _is_bare_directory(title: str, dirs: list[str]) -> bool:
+ """Whether the title is just the directory it covers, respelled.
+
+ ``core/ingestion`` and ``Ingestion`` are both failures of the same kind:
+ they tell a reader where the code is, which they can already see, instead
+ of what it does.
+ """
+ flat = title.strip().lower().replace("-", " ").replace("_", " ").replace("/", " ")
+ for d in dirs:
+ segs = [s for s in d.replace("-", " ").replace("_", " ").split("/") if s]
+ if not segs:
+ continue
+ if flat == " ".join(segs).lower() or flat == segs[-1].lower():
+ return True
+ return False
+
+
+def validate_outline(
+ outline: ConceptOutline,
+ *,
+ all_files: set[str],
+ test_files: set[str] | None = None,
+ max_files_per_page: int,
+) -> OutlineReport:
+ """Measure *outline* against the file set it claims to cover.
+
+ *all_files* is the production file set the grouper was given — the ground
+ truth for both coverage and fabrication. A member the outline names that is
+ not in it is an invented path, and the distinction between "invented" and
+ "unclaimed" matters: they have different causes and the probes moved them
+ in opposite directions.
+ """
+ report = OutlineReport()
+ pages = outline.pages
+ report.page_count = len(pages)
+ report.section_count = len(outline.sections)
+ report.total_files = len(all_files)
+
+ claimed: Counter[str] = Counter()
+ for page in pages:
+ if not page.members:
+ report.empty_pages.append(page.title or page.target_path)
+ for member in page.members:
+ claimed[member] += 1
+
+ known = set(claimed)
+ report.invented_paths = sorted(known - all_files)
+ report.covered_files = len(known & all_files)
+ report.unclaimed_files = sorted(all_files - known)
+ report.double_claimed_files = sorted(p for p, n in claimed.items() if n > 1)
+
+ if test_files:
+ report.test_paths_included = sorted(known & test_files)
+
+ for page in pages:
+ if len(page.members) > max_files_per_page and not page.group.oversized:
+ report.oversized_pages.append(f"{page.title} ({len(page.members)})")
+
+ for section in outline.sections:
+ if len(section.pages) == 1:
+ report.single_child_sections.append(section.title)
+
+ titles = [p.title.strip() for p in pages]
+ report.duplicate_titles = sorted(
+ t for t, n in Counter(t.lower() for t in titles).items() if n > 1
+ )
+ for page in pages:
+ words = len(page.title.split())
+ if words < MIN_TITLE_WORDS or words > MAX_TITLE_WORDS:
+ report.bad_length_titles.append(f"{page.title} ({words}w)")
+ if _is_bare_directory(page.title, page.group.dirs):
+ report.bare_directory_titles.append(page.title)
+ report.title_word_avg = (
+ sum(len(t.split()) for t in titles) / len(titles) if titles else 0.0
+ )
+
+ keys = Counter(p.structural_key for p in pages)
+ report.duplicate_structural_keys = sorted(k for k, n in keys.items() if n > 1)
+
+ # The page id is derived from the target path alone, so a repeat here is a
+ # row collision rather than a cosmetic one: the second page would overwrite
+ # the first on persist and the wiki would quietly lose a page.
+ targets = Counter(p.target_path for p in pages)
+ report.duplicate_target_paths = sorted(t for t, n in targets.items() if n > 1)
+
+ return report
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
new file mode 100644
index 000000000..71f748a3f
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
@@ -0,0 +1,245 @@
+"""The project's own words for its own subsystems, bound to real groups.
+
+A wiki that reuses the repository's terminology reads as one document instead
+of as a pile of pages. The naive way to get that is to hand the planner the
+README, and it is measured to be a bad idea: nine thousand characters of
+positioning prose alongside a structural task collapsed the outline from 23
+pages covering 87.5% of the code to three pages covering 5.4%, and it invented
+a docs path that matched the project's marketing language rather than its
+directory tree.
+
+So terminology enters through a keyhole. Terms are lifted from the repository's
+docs here, and a term only survives if it can be **bound to a group that
+structure already produced**. A feature that is marketed but not built has no
+group to bind to and therefore gets no page. Structure leads and vocabulary
+decorates, which is the only arrangement where a thin, stale or absent README —
+the common case in the wild — costs nothing structural.
+
+Extraction is deterministic rather than another model call. It has to run on
+the keyless path, it has to give the same answer twice, and the job is small
+enough that a model adds cost and variance without adding much.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from .grouping import ConceptGroup
+
+#: Documents worth mining, relative to the repository root. A doc that names a
+#: subsystem in a heading is a much stronger signal than one that mentions it
+#: in a sentence, so headings are what this reads.
+#:
+#: These are root-anchored on purpose. Glob patterns containing ``/`` match a
+#: path *tail*, so a pattern like ``docs/*.md`` also matches
+#: ``some/vendored/thing/docs/notes.md`` — which on this repository pulled
+#: planning notes and marketing copy into the glossary and bound terms like
+#: "Headline facts updated" to source directories.
+DOC_FILES: tuple[str, ...] = (
+ "README.md",
+ "README.rst",
+ "ARCHITECTURE.md",
+ "CONTRIBUTING.md",
+)
+
+#: Directories, relative to the root, whose top level is scanned for docs.
+DOC_DIRS: tuple[str, ...] = ("docs", "doc", "adr", "docs/adr", "docs/decisions")
+
+_MAX_DOC_BYTES = 200_000
+_MAX_DOCS = 60
+
+# A markdown or rst heading. The term is the heading text.
+_HEADING = re.compile(r"^\s{0,3}#{1,4}\s+(.+?)\s*#*\s*$", re.MULTILINE)
+# Bolded lead-ins are how glossaries are usually written: "**Blast radius** — ".
+_BOLD_TERM = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*")
+
+_STOPWORDS = frozenset(
+ {
+ "the", "a", "an", "and", "or", "of", "for", "to", "in", "on", "with",
+ "how", "what", "why", "when", "is", "are", "it", "this", "that", "you",
+ "your", "we", "our", "using", "use", "usage", "getting", "started",
+ "installation", "install", "quickstart", "quick", "start", "license",
+ "contributing", "changelog", "table", "contents", "overview",
+ "introduction", "intro", "example", "examples", "faq", "notes",
+ "requirements", "setup", "configuration", "config", "options", "api",
+ "reference", "guide", "tutorial", "docs", "documentation", "features",
+ "roadmap", "credits", "acknowledgements", "badges", "support",
+ }
+)
+
+_MIN_TERM_WORDS = 1
+_MAX_TERM_WORDS = 4
+
+# A heading that is a sentence is describing something, not naming it.
+# "Local paths are masked" and "Headline facts updated 2026-06-22" are both
+# real headings from this repository's own docs, and neither is a subsystem.
+_SENTENCE_VERBS = frozenset(
+ {
+ "is", "are", "was", "were", "has", "have", "had", "do", "does", "did",
+ "can", "will", "would", "should", "must", "may", "updated", "added",
+ "removed", "fixed", "works", "means", "needs", "gets", "goes", "runs",
+ "returns", "uses", "supports", "requires", "shows", "makes", "keeps",
+ }
+)
+
+
+def _normalise(term: str) -> str:
+ term = re.sub(r"`([^`]*)`", r"\1", term)
+ term = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", term) # markdown links
+ term = re.sub(r"[^\w\s/&+-]", " ", term)
+ term = re.sub(r"\s+", " ", term).strip()
+ return term
+
+
+def _is_useful(term: str) -> bool:
+ words = term.split()
+ if not (_MIN_TERM_WORDS <= len(words) <= _MAX_TERM_WORDS):
+ return False
+ if all(w.lower() in _STOPWORDS for w in words):
+ return False
+ # A heading that is entirely boilerplate ("Getting Started") says nothing
+ # about this repository in particular.
+ meaningful = [w for w in words if w.lower() not in _STOPWORDS]
+ if not meaningful:
+ return False
+ if not any(len(w) > 2 for w in meaningful):
+ return False
+ # Sentences and dated notes are prose, not names.
+ if any(w.lower() in _SENTENCE_VERBS for w in words):
+ return False
+ if any(any(ch.isdigit() for ch in w) for w in words):
+ return False
+ # A name starts like a name. All-lowercase multi-word headings are almost
+ # always prose fragments.
+ return len(words) == 1 or words[0][:1].isupper()
+
+
+def extract_terms(repo_root: Path, *, max_terms: int = 200) -> list[str]:
+ """Lift candidate subsystem names from the repository's own documentation.
+
+ Returns terms in document order with duplicates removed. Never raises: a
+ repository with no docs returns an empty list, which is a supported and
+ common outcome, not a degraded one.
+ """
+ seen: set[str] = set()
+ terms: list[str] = []
+ files: list[Path] = []
+ try:
+ for name in DOC_FILES:
+ candidate = repo_root / name
+ if candidate.is_file():
+ files.append(candidate)
+ for rel_dir in DOC_DIRS:
+ directory = repo_root / rel_dir
+ if not directory.is_dir():
+ continue
+ for candidate in sorted(directory.glob("*.md")):
+ if candidate.is_file() and candidate not in files:
+ files.append(candidate)
+ if len(files) >= _MAX_DOCS:
+ break
+ if len(files) >= _MAX_DOCS:
+ break
+ except OSError:
+ return []
+
+ for path in files:
+ try:
+ text = path.read_text(encoding="utf-8", errors="replace")[:_MAX_DOC_BYTES]
+ except OSError:
+ continue
+ for match in list(_HEADING.finditer(text)) + list(_BOLD_TERM.finditer(text)):
+ term = _normalise(match.group(1))
+ if not _is_useful(term):
+ continue
+ key = term.lower()
+ if key in seen:
+ continue
+ seen.add(key)
+ terms.append(term)
+ if len(terms) >= max_terms:
+ return terms
+ return terms
+
+
+# ---------------------------------------------------------------------------
+# Binding
+# ---------------------------------------------------------------------------
+
+
+def _tokens(text: str) -> set[str]:
+ parts = re.split(r"[^A-Za-z0-9]+", text)
+ out: set[str] = set()
+ for part in parts:
+ if not part:
+ continue
+ for piece in re.findall(r"[A-Z]+(?![a-z])|[A-Z][a-z]*|[a-z]+|\d+", part) or [part]:
+ low = piece.lower()
+ if len(low) > 2 and low not in _STOPWORDS:
+ out.add(low)
+ # A trailing plural should still match its directory.
+ if low.endswith("s"):
+ out.add(low[:-1])
+ return out
+
+
+def _group_tokens(group: ConceptGroup) -> tuple[set[str], set[str]]:
+ """The group's words, split into structural ones and incidental ones.
+
+ Directory names are what the codebase calls a *place*; filenames are what
+ it calls the things in that place. A term matching only a filename is
+ usually a coincidence — every group contains a file with "client" or
+ "store" in its name — so the two are scored differently.
+ """
+ structural = _tokens(group.target_path)
+ for d in group.dirs:
+ structural |= _tokens(d)
+ incidental: set[str] = set()
+ for member in group.members:
+ incidental |= _tokens(member.rsplit("/", 1)[-1].rsplit(".", 1)[0])
+ return structural, incidental - structural
+
+
+def bind_terms(
+ terms: list[str],
+ groups: dict[str, ConceptGroup],
+) -> dict[str, str]:
+ """Bind each term to at most one group, and each group to at most one term.
+
+ A term binds only when the codebase spells **all** of it and at least part
+ of it in a directory name. Both halves matter. Requiring every word keeps
+ "Change Risk" off a group that merely contains the word "change";
+ requiring a directory hit keeps a term from attaching to whichever group
+ happens to hold a file with a matching name. A marketed feature with no
+ code cluster satisfies neither and therefore never becomes a page title,
+ which is the entire point of binding rather than injecting.
+
+ Returns ``{group_id: term}``. Ties break on how much of the match was
+ structural, then the term's document order, then the group id, so two runs
+ over an unchanged repository bind identically.
+ """
+ token_cache = {gid: _group_tokens(g) for gid, g in groups.items()}
+ scored: list[tuple[int, int, int, str, str]] = []
+ for order, term in enumerate(terms):
+ wanted = _tokens(term)
+ if not wanted:
+ continue
+ for gid, (structural, incidental) in token_cache.items():
+ in_struct = wanted & structural
+ if not in_struct:
+ continue
+ if wanted - structural - incidental:
+ # Some word of the term is absent from this group entirely.
+ continue
+ scored.append((-len(in_struct), -len(wanted), order, gid, term))
+
+ scored.sort()
+ bound: dict[str, str] = {}
+ used_terms: set[str] = set()
+ for _s, _w, _order, gid, term in scored:
+ if gid in bound or term.lower() in used_terms:
+ continue
+ bound[gid] = term
+ used_terms.add(term.lower())
+ return bound
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index 6f15e2af0..dfff264c3 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -750,10 +750,20 @@ def _stamp_structural_keys(pages: list[GeneratedPage]) -> None:
A page keyed on a real file path has no identity separate from that path,
so it is left unset rather than given a copy of the path.
+
+ A key already set by whatever produced the page is kept. The concept-tree
+ planner computes its groups' identities as part of deciding what the
+ groups *are*, and hashes exactly the same member list this would; letting
+ the stamp recompute it would give two places that must agree about page
+ identity, which is the arrangement D2 exists to avoid. It also lets the
+ key say which algorithm produced the page, so a wiki holding both the old
+ per-directory modules and the new concept groups can tell them apart.
"""
for page in pages:
if page.page_type not in STRUCTURALLY_KEYED_PAGE_TYPES:
continue
+ if page.structural_key:
+ continue
prefix = _MEMBER_KEYED_PREFIX.get(page.page_type)
members = page.metadata.get("file_paths") or []
members = [m for m in members if isinstance(m, str)]
diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.py
index bb2bb4ab7..af79c25f3 100644
--- a/packages/core/src/repowise/core/pipeline/persist.py
+++ b/packages/core/src/repowise/core/pipeline/persist.py
@@ -598,6 +598,100 @@ async def _sweep_stale_generated_pages(
return swept
+async def sweep_superseded_generated_pages(
+ session: Any,
+ repo_id: str,
+ generated_pages: list[Any] | None,
+) -> list[str]:
+ """Retire structurally-keyed rows whose coverage a scoped run just took over.
+
+ :func:`_sweep_stale_generated_pages` cannot be used on the update path. It
+ deletes every row of a page type the run did not reproduce, which is right
+ for a full index and catastrophic for a scoped one: regenerating a single
+ page would wipe the other fifty.
+
+ That left a real gap. A structurally-keyed page's id is its
+ ``target_path``, and a page that groups files can legitimately change which
+ directory names it when its membership changes. ``repowise update`` goes
+ through ``scoped_generation`` and never swept, so the regenerated page was
+ written under its new id and the old row stayed behind as a duplicate until
+ somebody ran a full index. Harmless while module and cycle pages were never
+ regenerated incrementally, and not harmless once concept pages are.
+
+ The rule here is narrower and does not need to know what the caller asked
+ for: a prior row is superseded when **every file it covered is now covered
+ by pages this run produced**, of the same type. That is exactly the
+ condition under which the old row documents nothing that is not documented
+ elsewhere. A page that failed to generate keeps files no produced page
+ claims, so it survives — which matters, because deleting a page because its
+ generation errored would be a worse bug than the one this fixes.
+
+ Returns the swept page ids so the caller can drop them from FTS after the
+ session closes.
+ """
+ from sqlalchemy import delete, select
+
+ from repowise.core.persistence.models import Page, PageVersion
+
+ produced_ids: dict[str, set[str]] = {}
+ covered: dict[str, set[str]] = {}
+ for page in generated_pages or []:
+ page_type = page.page_type
+ if page_type not in _SWEPT_GENERATED_PAGE_TYPES:
+ continue
+ produced_ids.setdefault(page_type, set()).add(page.page_id)
+ metadata = getattr(page, "metadata", None) or {}
+ members = metadata.get("file_paths") or metadata.get("files") or []
+ covered.setdefault(page_type, set()).update(
+ m for m in members if isinstance(m, str)
+ )
+
+ swept: list[str] = []
+ for page_type, current in produced_ids.items():
+ union = covered.get(page_type) or set()
+ if not union:
+ # Nothing to compare against: a run that recorded no membership
+ # cannot prove it superseded anything, so it retires nothing.
+ continue
+ rows = (
+ await session.execute(
+ select(Page.id, Page.metadata_json).where(
+ Page.repository_id == repo_id, Page.page_type == page_type
+ )
+ )
+ ).all()
+ stale: list[str] = []
+ for page_id, raw in rows:
+ if page_id in current:
+ continue
+ try:
+ metadata = json.loads(raw) if isinstance(raw, str) else (raw or {})
+ except (TypeError, ValueError):
+ continue
+ if not isinstance(metadata, dict):
+ continue
+ members = metadata.get("file_paths") or metadata.get("files") or []
+ members = {m for m in members if isinstance(m, str)}
+ # No recorded membership means no proof of supersession. Those rows
+ # are the pre-membership ones (backlog B20) and they are left for a
+ # full index to retire.
+ if members and members <= union:
+ stale.append(page_id)
+ for i in range(0, len(stale), _PRUNE_CHUNK):
+ batch = stale[i : i + _PRUNE_CHUNK]
+ await session.execute(delete(PageVersion).where(PageVersion.page_id.in_(batch)))
+ await session.execute(
+ delete(Page).where(Page.repository_id == repo_id, Page.id.in_(batch))
+ )
+ swept.extend(stale)
+
+ if swept:
+ logger.info(
+ "superseded_generated_pages_swept", repo_id=repo_id, count=len(swept)
+ )
+ return swept
+
+
async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:
"""Persist ingestion-phase outputs: graph nodes/edges, external systems,
symbols, and the per-file security scan.
diff --git a/packages/core/src/repowise/core/pipeline/scoped_generation.py b/packages/core/src/repowise/core/pipeline/scoped_generation.py
index 118be7bff..232fe859b 100644
--- a/packages/core/src/repowise/core/pipeline/scoped_generation.py
+++ b/packages/core/src/repowise/core/pipeline/scoped_generation.py
@@ -26,7 +26,7 @@ code, so hosted reuses it as-is.
from __future__ import annotations
import asyncio
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
@@ -67,6 +67,10 @@ class ScopedGenerationResult:
generated_pages: list[Any]
marked_stale: int
+ #: Rows retired because pages this run produced now cover everything they
+ #: covered. Reported so an update can say it removed a duplicate rather
+ #: than doing it silently.
+ swept_page_ids: list[str] = field(default_factory=list)
def load_kg_context(repo_path: Path) -> Any:
@@ -202,7 +206,10 @@ async def execute_scoped_generation(
from repowise.core.persistence import get_session, upsert_pages_from_generated
from repowise.core.persistence.crud import backfill_related_pages
from repowise.core.pipeline import run_generation
- from repowise.core.pipeline.persist import mark_page_ids_stale
+ from repowise.core.pipeline.persist import (
+ mark_page_ids_stale,
+ sweep_superseded_generated_pages,
+ )
# Bill this run's LLM calls to the caller's tracker. run_generation reads the
# passed tracker; some providers additionally consult ``_cost_tracker``.
@@ -229,11 +236,22 @@ async def execute_scoped_generation(
await cost_tracker.flush()
marked_stale = 0
+ swept_page_ids: list[str] = []
async with get_session(session_factory) as session:
await upsert_pages_from_generated(session, generated_pages, repo_id)
+ # A page whose identity is its members can change which directory names
+ # it when those members change, and the new row is written under a new
+ # id. Without this the old row survives as a duplicate until somebody
+ # runs a full index, because the full sweep cannot run here — it would
+ # delete every page of a type this run did not reproduce, which on a
+ # scoped run is nearly all of them.
+ swept_page_ids = await sweep_superseded_generated_pages(
+ session, repo_id, generated_pages
+ )
# A scoped run holds only the pages it was asked for, so it cannot
# work out where they sit. Placement comes from the store, which has
# the whole set; without this the pages it touched lose their place.
+ # After the sweep, so a retired row cannot be given a place.
from .page_tree_sync import rebuild_page_tree
await rebuild_page_tree(session, repo_id)
@@ -260,8 +278,22 @@ async def execute_scoped_generation(
except Exception as exc:
logger.debug("related_pages_backfill_skipped", error=str(exc))
+ # Same ordering rule the full-index path follows: the vector delete
+ # goes before the SQL commit, because the store is a separate engine
+ # and an interrupted run then self-heals rather than leaving an
+ # embedding for a row that no longer exists.
+ if swept_page_ids and vector_store is not None:
+ try:
+ await vector_store.delete_many(swept_page_ids)
+ except Exception as exc:
+ logger.debug("vector_sweep_skipped", error=str(exc))
+
+ # FTS lives in the same SQLite file as the session, so it is only safe to
+ # touch after that session has closed.
if fts is not None:
try:
+ if swept_page_ids:
+ await fts.delete_many(swept_page_ids)
for page in generated_pages:
await fts.index(page.page_id, page.title, page.content)
except Exception as exc:
@@ -270,4 +302,5 @@ async def execute_scoped_generation(
return ScopedGenerationResult(
generated_pages=generated_pages,
marked_stale=marked_stale,
+ swept_page_ids=swept_page_ids,
)
diff --git a/tests/unit/generation/test_concept_grouping.py b/tests/unit/generation/test_concept_grouping.py
new file mode 100644
index 000000000..1720496e8
--- /dev/null
+++ b/tests/unit/generation/test_concept_grouping.py
@@ -0,0 +1,347 @@
+"""The deterministic partition: totality, stability, and bounded page size.
+
+Grouping is the half of the outline planner that must not vary, because page
+identity hashes the member list. A grouping that reshuffled between two runs of
+the same commit would remint every page id, strand the vectors behind them, and
+leave the old rows as duplicates. So the properties asserted here are
+correctness properties, not tidiness ones.
+
+Every fixture that depends on ordering is built so the structure disagrees with
+the alphabet. A fixture whose spine and alphabet agree cannot tell a test that
+reads structure from one that sorts.
+"""
+
+from __future__ import annotations
+
+import os
+
+from repowise.core.generation.concept_tree.grouping import (
+ GroupingParams,
+ group_files,
+ params_for,
+)
+
+# Two sibling subsystems plus a deep one. Named so alphabetical order
+# (alpha, beta, zulu) is NOT the order the tree nests them in.
+TINY = GroupingParams(min_files=3, max_files=6)
+
+
+def _repo(spec: dict[str, int]) -> list[str]:
+ """Build a file list: ``{directory: count}``. ``""`` is the repo root."""
+ out: list[str] = []
+ for directory, count in spec.items():
+ prefix = f"{directory}/" if directory else ""
+ out.extend(f"{prefix}f{i:02d}.py" for i in range(count))
+ return out
+
+
+SPEC = {
+ # Root-level files are the common case, not an edge case: setup.py,
+ # manage.py, index.js, vite.config.ts all live here. A fixture without them
+ # cannot see that the root's directory path is the empty string, which is
+ # a real directory and was once mistaken for a missing one.
+ "": 3,
+ "src/zulu": 5,
+ "src/alpha": 5,
+ "src/beta/deep": 5,
+ "src/beta/other": 4,
+}
+
+
+class TestTotality:
+ def test_every_file_lands_in_exactly_one_group(self):
+ files = _repo(SPEC)
+ groups = group_files(files, params=TINY)
+ claimed = [m for g in groups for m in g.members]
+ assert sorted(claimed) == sorted(files)
+ assert len(claimed) == len(set(claimed))
+
+ def test_no_group_is_empty(self):
+ """An empty group is a page about nothing, and a hard validator failure."""
+ groups = group_files(_repo(SPEC), params=TINY)
+ assert groups
+ assert all(g.members for g in groups)
+
+ def test_empty_input_is_harmless(self):
+ assert group_files([]) == []
+
+ def test_a_repo_smaller_than_the_ceiling_is_one_group(self):
+ groups = group_files(_repo({"src": 4}), params=TINY)
+ assert len(groups) == 1
+ assert groups[0].file_count == 4
+
+
+class TestStability:
+ def test_two_processes_produce_identical_structural_keys(self):
+ """Determinism that survives a fresh interpreter, not just a fresh call.
+
+ Calling a pure function twice in one process cannot detect the failure
+ that actually matters here. Python randomises string hashing per
+ process, so a grouping that leaked set or dict iteration order would
+ agree with itself all day and disagree between two indexing runs —
+ reminting every page id. These two subprocesses are given different
+ hash seeds on purpose.
+ """
+ import json
+ import subprocess
+ import sys
+ import textwrap
+
+ script = textwrap.dedent(
+ """
+ import json
+ from repowise.core.generation.concept_tree.grouping import (
+ GroupingParams, group_files,
+ )
+ spec = json.loads(input())
+ files = []
+ for d, n in spec.items():
+ p = f"{d}/" if d else ""
+ files.extend(f"{p}f{i:02d}.py" for i in range(n))
+ groups = group_files(files, params=GroupingParams(3, 6))
+ print(json.dumps([[g.target_path, g.structural_key] for g in groups]))
+ """
+ )
+
+ def run(seed: str) -> list[list[str]]:
+ out = subprocess.run(
+ [sys.executable, "-c", script],
+ input=json.dumps(SPEC),
+ capture_output=True,
+ text=True,
+ env={**os.environ, "PYTHONHASHSEED": seed},
+ check=True,
+ )
+ return json.loads(out.stdout.strip().splitlines()[-1])
+
+ assert run("0") == run("12345")
+
+ def test_input_order_does_not_change_the_grouping(self):
+ files = _repo(SPEC)
+ forward = [g.structural_key for g in group_files(files, params=TINY)]
+ reverse = [
+ g.structural_key for g in group_files(list(reversed(files)), params=TINY)
+ ]
+ assert forward == reverse
+
+ def test_one_added_file_does_not_reshuffle_the_tree(self):
+ """The property the whole design exists for.
+
+ A commit that adds one file must change the identity of the page that
+ gained it and of nothing else. If a global parameter were derived from
+ the file count, every key would move on every commit.
+ """
+ files = _repo(SPEC)
+ before = {g.structural_key for g in group_files(files, params=TINY)}
+ after = {
+ g.structural_key
+ for g in group_files([*files, "src/alpha/new_thing.py"], params=TINY)
+ }
+ assert len(before - after) == 1, "more than one group changed identity"
+ assert len(after - before) == 1
+
+ def test_the_size_ladder_is_flat_across_a_whole_band(self):
+ """No parameter may be computed from the file count.
+
+ A ceiling of ``total / target`` would move on almost every commit, and
+ because the ceiling decides the partition, every page in the wiki
+ would be reminted. Asserting two adjacent counts is not enough to catch
+ that — integer division agrees with itself most of the time — so this
+ asserts the whole band is flat.
+ """
+ band = [params_for(n) for n in range(1250, 2350, 7)]
+ assert len(set(band)) == 1, "the ceiling varies with the file count"
+
+ def test_the_ladder_does_step_between_bands(self):
+ """The flatness above must not be flatness everywhere."""
+ assert params_for(500) != params_for(5000)
+
+ def test_an_unrelated_group_keeps_its_identity_when_another_changes(self):
+ files = _repo(SPEC)
+ base = {g.target_path: g.structural_key for g in group_files(files, params=TINY)}
+ after = {
+ g.target_path: g.structural_key
+ for g in group_files([*files, "src/alpha/new_thing.py"], params=TINY)
+ }
+ untouched = [t for t in base if "alpha" not in t and t in after]
+ assert untouched, "fixture proves nothing if no group is untouched"
+ for target in untouched:
+ assert base[target] == after[target]
+
+
+class TestIdentity:
+ def test_target_paths_are_unique(self):
+ """Two groups sharing a target path would share a page id."""
+ groups = group_files(_repo(SPEC), params=TINY)
+ targets = [g.target_path for g in groups]
+ assert len(targets) == len(set(targets))
+
+ def test_two_runs_of_siblings_under_one_parent_do_not_collide(self):
+ """The case that actually produced a page-id collision.
+
+ Four sibling directories partition into two groups of two. Neither
+ group holds a file sitting directly in the shared parent, so both
+ compute that parent as their shallowest common directory — and the
+ page id is the target path, so the second page would have overwritten
+ the first on persist. Verified to fail when target assignment is
+ removed.
+ """
+ params = GroupingParams(min_files=3, max_files=9)
+ files = _repo({"src/x/a": 4, "src/x/b": 4, "src/x/c": 4, "src/x/d": 4})
+ groups = group_files(files, params=params)
+ assert len(groups) > 1, "fixture must produce more than one group"
+ targets = [g.target_path for g in groups]
+ assert len(targets) == len(set(targets)), targets
+
+ def test_target_path_stays_a_real_directory(self):
+ files = _repo(SPEC)
+ directories = {f.rsplit("/", 1)[0] if "/" in f else "" for f in files}
+ for group in group_files(files, params=TINY):
+ assert group.target_path in directories, (
+ f"{group.target_path!r} is not a directory in this repository"
+ )
+
+ def test_a_group_of_root_level_files_is_targeted_at_the_root(self):
+ """The repository root is a directory whose path is the empty string.
+
+ Treating that as "no directory found" sent the group to the
+ identity-hash fallback, which then became its page id, its title
+ ("D 73789 F 001 Bd") and its scope sentence.
+ """
+ files = _repo({"": 3, "src/pkg": 10})
+ groups = group_files(files, params=GroupingParams(min_files=2, max_files=4))
+ root = next(g for g in groups if "setup" in " ".join(g.members) or "f00.py" in g.members)
+ assert root.target_path == ""
+ assert "#" not in root.target_path
+ assert all("#" not in g.target_path for g in groups)
+
+ def test_structural_key_follows_members_not_target(self):
+ a = group_files(_repo({"src/one": 4}), params=TINY)[0]
+ b = group_files(_repo({"src/two": 4}), params=TINY)[0]
+ # Different paths, same shape: the key must differ because the members
+ # differ, not because the directory name does.
+ assert a.structural_key != b.structural_key
+
+ def test_reordering_members_does_not_change_the_key(self):
+ files = _repo({"src/one": 4})
+ a = group_files(files, params=TINY)[0]
+ b = group_files(list(reversed(files)), params=TINY)[0]
+ assert a.structural_key == b.structural_key
+
+
+class TestSize:
+ def test_no_group_exceeds_the_ceiling_unless_it_is_one_directory(self):
+ """The fixture must actually contain an over-ceiling group.
+
+ It did not, once: every group in ``SPEC`` fits under the ceiling, so
+ the loop body never ran and the test passed whatever the code did.
+ """
+ spec = {**SPEC, "src/huge": TINY.max_files * 3}
+ groups = group_files(_repo(spec), params=TINY)
+ over = [g for g in groups if g.file_count > TINY.max_files]
+ assert over, "fixture produced no over-ceiling group, so this proves nothing"
+ for group in over:
+ assert group.oversized
+ assert len(group.dirs) == 1
+
+ def test_a_flat_directory_over_the_ceiling_is_flagged_not_split(self):
+ """No path rule can split it, so it stays whole and says so."""
+ groups = group_files(_repo({"src/big": 20}), params=TINY)
+ assert len(groups) == 1
+ assert groups[0].oversized
+ assert groups[0].file_count == 20
+
+ def test_absorption_considers_the_left_neighbour_too(self):
+ """A unit test of the absorption pass, not of a whole partition.
+
+ The walk merges forward only: a thin remainder is offered to the group
+ it was flushed after. Absorption is what gives a thin group its other
+ neighbour, and the case is reachable but fiddly to provoke end to end,
+ so it is asserted directly. Here the right neighbour is full and only
+ the left one has room.
+ """
+ from repowise.core.generation.concept_tree.grouping import (
+ _absorb_thin,
+ _Partitioner,
+ )
+
+ params = GroupingParams(min_files=4, max_files=8)
+ part = _Partitioner(params, {})
+ groups = [
+ part.make(_repo({"src/a": 5})),
+ part.make(_repo({"src/b": 1})),
+ part.make(_repo({"src/c": 8})),
+ ]
+ result = _absorb_thin(groups, part)
+ assert [g.file_count for g in result] == [6, 8]
+ assert not any(g.file_count < params.min_files for g in result)
+
+ def test_grouping_actually_runs_the_absorption_pass(self, monkeypatch):
+ """Wiring, asserted separately from behaviour.
+
+ The pass above can be correct and simply not called. The end-to-end
+ fixtures cannot catch that, because the forward merge in the walk
+ already absorbs most thin remainders, so removing the call leaves
+ their output unchanged.
+ """
+ from repowise.core.generation.concept_tree import grouping as mod
+
+ called: list[int] = []
+
+ def spy(groups, part):
+ called.append(len(groups))
+ return groups
+
+ monkeypatch.setattr(mod, "_absorb_thin", spy)
+ mod.group_files(_repo(SPEC), params=TINY)
+ assert called, "group_files did not run the absorption pass"
+
+
+class TestLayers:
+ def test_the_layer_boundary_changes_which_siblings_merge(self):
+ """The layer signal steers a split that is already happening.
+
+ Without it the walk merges the first two siblings that fit; with it,
+ the first sibling is flushed alone because its layer differs, and the
+ two that share a layer merge instead. Asserting composition rather
+ than group count is what makes this discriminate: both answers have
+ two groups.
+ """
+ params = GroupingParams(min_files=3, max_files=12)
+ files = _repo({"src/aaa": 5, "src/bbb": 5, "src/ccc": 5})
+ layers = {f: ("layer:ui" if "/aaa/" in f else "layer:core") for f in files}
+
+ blind = {g.target_path: g.file_count for g in group_files(files, params=params)}
+ aware = {
+ g.target_path: g.file_count
+ for g in group_files(files, layer_of_file=layers, params=params)
+ }
+ # Blind: aaa and bbb merge, ccc is flushed on its own.
+ assert blind == {"src/aaa": 10, "src/ccc": 5}
+ # Aware: aaa is flushed alone at the layer boundary, bbb and ccc merge.
+ assert aware == {"src/aaa": 5, "src/bbb": 10}
+ assert len(blind) == len(aware), "counts match, so composition is the signal"
+
+ def test_a_subtree_within_the_ceiling_is_not_split_by_layer(self):
+ """A documented limitation, asserted so it cannot change silently.
+
+ Recursion stops as soon as a subtree fits, so the layer signal is
+ never consulted for one that does — a small directory spanning two
+ layers stays one page. Splitting every fitting subtree on layer would
+ fragment the outline and would inherit the layer data's own noise,
+ which is worse than an occasionally impure small page.
+ """
+ params = GroupingParams(min_files=3, max_files=12)
+ files = _repo({"src/aaa": 5, "src/bbb": 5})
+ layers = {f: ("layer:ui" if "/aaa/" in f else "layer:core") for f in files}
+ groups = group_files(files, layer_of_file=layers, params=params)
+ assert len(groups) == 1
+
+ def test_dominant_layer_ties_break_deterministically(self):
+ files = _repo({"src/x": 4})
+ layers = {f: ("layer:a" if i % 2 else "layer:b") for i, f in enumerate(files)}
+ first = group_files(files, layer_of_file=layers, params=TINY)[0]
+ second = group_files(
+ list(reversed(files)), layer_of_file=layers, params=TINY
+ )[0]
+ assert first.dominant_layer == second.dominant_layer
diff --git a/tests/unit/generation/test_concept_outline.py b/tests/unit/generation/test_concept_outline.py
new file mode 100644
index 000000000..35ad33e9e
--- /dev/null
+++ b/tests/unit/generation/test_concept_outline.py
@@ -0,0 +1,346 @@
+"""The planner: what the model can and cannot break, and what the validator sees.
+
+The design claim this file exists to hold is narrow and testable: **naming is
+the only thing the model decides**. Coverage, membership and page identity come
+from the deterministic partition, so a model that skips half the groups, echoes
+back ids that were never in the input, or returns nothing at all still leaves a
+complete and correctly-identified outline behind.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.generation.concept_tree.grouping import (
+ GroupingParams,
+ group_files,
+)
+from repowise.core.generation.concept_tree.naming import (
+ build_payload,
+ decode_response,
+ deterministic_title,
+)
+from repowise.core.generation.concept_tree.planner import PlannerInputs, plan_outline
+from repowise.core.generation.concept_tree.validation import validate_outline
+
+TINY = GroupingParams(min_files=3, max_files=6)
+
+FILES = [
+ "src/ingest/reader.py",
+ "src/ingest/parser.py",
+ "src/ingest/walker.py",
+ "src/ingest/cache.py",
+ "src/render/html.py",
+ "src/render/markdown.py",
+ "src/render/theme.py",
+ "src/render/layout.py",
+]
+TESTS = {"tests/test_reader.py"}
+
+
+def _inputs() -> PlannerInputs:
+ return PlannerInputs(
+ repo_name="demo",
+ production_files=list(FILES),
+ layer_labels={"layer:core": "Core"},
+ test_files=set(TESTS),
+ )
+
+
+class FakeProvider:
+ """A provider whose reply is whatever the test needs it to be."""
+
+ def __init__(self, content: str):
+ self.content = content
+ self.calls: list[str] = []
+
+ async def generate(self, *, system_prompt, user_prompt, **kwargs):
+ self.calls.append(user_prompt)
+
+ class R:
+ content = self.content
+
+ return R()
+
+
+class TestModelCannotBreakStructure:
+ @pytest.mark.asyncio
+ async def test_a_model_that_returns_nothing_still_covers_every_file(self):
+ _outline, report = await plan_outline(
+ _inputs(), provider=FakeProvider(""), params=TINY, repair=False
+ )
+ assert report.coverage == 1.0
+ assert report.unclaimed_files == []
+ assert report.ok
+
+ @pytest.mark.asyncio
+ async def test_a_model_that_invents_group_ids_does_not_add_pages(self):
+ groups = group_files(FILES, params=TINY)
+ _payload, index = build_payload(groups)
+ data = {
+ "sections": [
+ {"title": "Made Up", "groups": ["g99", "gzz"]},
+ {"title": "Real", "groups": list(index)},
+ ],
+ "names": {"g99": {"title": "Ghost Page", "scope": "x"}},
+ }
+ named, invented = decode_response(data, index)
+ assert invented == ["g99", "gzz"]
+ assert len(named) == len(index)
+ assert "Ghost Page" not in [n.title for n in named]
+
+ @pytest.mark.asyncio
+ async def test_a_model_that_skips_groups_leaves_them_named_anyway(self):
+ groups = group_files(FILES, params=TINY)
+ _payload, index = build_payload(groups)
+ first = next(iter(index))
+ data = {"sections": [], "names": {first: {"title": "Real Title", "scope": "s"}}}
+ named, _ = decode_response(data, index)
+ assert len(named) == len(index)
+ skipped = [n for n in named if n.fallback]
+ assert skipped, "fixture must leave at least one group unnamed"
+ assert all(n.title for n in named)
+
+ def test_decode_survives_a_non_list_group_field(self):
+ """A model asked for a list of ids returned a count instead.
+
+ ``section.get("groups") or []`` passes a non-empty scalar straight
+ through to the loop, which raised. Asserted against ``decode_response``
+ directly, because the planner also wraps this call and a test that only
+ went through the planner would pass on the safety net alone.
+ """
+ groups = group_files(FILES, params=TINY)
+ _payload, index = build_payload(groups)
+ named, invented = decode_response(
+ {"sections": [{"title": "Core", "groups": 3}], "names": {}}, index
+ )
+ assert len(named) == len(index)
+ assert invented == []
+
+ def test_decode_ignores_group_ids_that_are_not_scalars(self):
+ groups = group_files(FILES, params=TINY)
+ _payload, index = build_payload(groups)
+ named, _ = decode_response(
+ {"sections": [{"title": "Core", "groups": [{"a": 1}, ["b"], None]}],
+ "names": {}},
+ index,
+ )
+ assert len(named) == len(index)
+
+ @pytest.mark.asyncio
+ async def test_a_non_list_group_field_does_not_crash_the_run(self):
+ """The same malformed response, end to end through the planner."""
+ _outline, report = await plan_outline(
+ _inputs(),
+ provider=FakeProvider('{"sections":[{"title":"Core","groups":3}],"names":{}}'),
+ params=TINY,
+ repair=False,
+ )
+ assert report.coverage == 1.0
+ assert report.ok
+
+ @pytest.mark.asyncio
+ async def test_junk_types_in_the_response_are_ignored(self):
+ content = (
+ '{"sections": {"not": "a list"},'
+ ' "names": {"g01": ["not", "a", "dict"], "g02": 7}}'
+ )
+ outline, report = await plan_outline(
+ _inputs(), provider=FakeProvider(content), params=TINY, repair=False
+ )
+ assert report.coverage == 1.0
+ assert report.ok
+ assert all(p.title for p in outline.pages)
+
+ @pytest.mark.asyncio
+ async def test_an_enormous_title_is_bounded(self):
+ groups = group_files(FILES, params=TINY)
+ _payload, index = build_payload(groups)
+ gid = next(iter(index))
+ data = {"sections": [], "names": {gid: {"title": "Wildly " * 400, "scope": "s"}}}
+ named, _ = decode_response(data, index)
+ assert all(len(n.title) <= 120 for n in named)
+
+ @pytest.mark.asyncio
+ async def test_a_model_claiming_one_group_twice_places_it_once(self):
+ groups = group_files(FILES, params=TINY)
+ _payload, index = build_payload(groups)
+ gid = next(iter(index))
+ data = {
+ "sections": [
+ {"title": "First", "groups": [gid]},
+ {"title": "Second", "groups": [gid]},
+ ],
+ "names": {},
+ }
+ named, _ = decode_response(data, index)
+ assert sum(1 for n in named if n.group is index[gid]) == 1
+
+
+class TestDeterministicPath:
+ @pytest.mark.asyncio
+ async def test_no_provider_still_produces_a_valid_outline(self):
+ outline, report = await plan_outline(_inputs(), provider=None, params=TINY)
+ assert outline.naming_mode == "deterministic"
+ assert report.coverage == 1.0
+ assert report.ok
+
+ @pytest.mark.asyncio
+ async def test_deterministic_mode_ignores_a_provider_that_is_present(self):
+ provider = FakeProvider('{"sections":[],"names":{}}')
+ outline, _ = await plan_outline(
+ _inputs(), provider=provider, deterministic=True, params=TINY
+ )
+ assert provider.calls == []
+ assert outline.naming_mode == "deterministic"
+
+ @pytest.mark.asyncio
+ async def test_the_keyless_tree_has_the_same_shape_as_the_named_one(self):
+ """D5: adding a key changes the prose, never the partition."""
+ keyless, _ = await plan_outline(_inputs(), provider=None, params=TINY)
+ named, _ = await plan_outline(
+ _inputs(), provider=FakeProvider(""), params=TINY, repair=False
+ )
+ assert sorted(p.structural_key for p in keyless.pages) == sorted(
+ p.structural_key for p in named.pages
+ )
+
+ def test_deterministic_titles_are_not_bare_directory_names(self):
+ """The name of this test is what it must actually assert.
+
+ A word-count bound alone would pass on "src ingest", which is the
+ exact failure the validator's bare-directory check exists to catch.
+ """
+ from repowise.core.generation.concept_tree.validation import (
+ _is_bare_directory,
+ )
+
+ for group in group_files(FILES, params=TINY):
+ title = deterministic_title(group)
+ assert 2 <= len(title.split()) <= 7
+ assert not _is_bare_directory(title, group.dirs), title
+
+
+class TestValidator:
+ def _outline(self):
+ from repowise.core.generation.concept_tree.models import (
+ ConceptOutline,
+ ConceptPage,
+ ConceptSection,
+ )
+
+ groups = group_files(FILES, params=TINY)
+ outline = ConceptOutline()
+ outline.sections.append(
+ ConceptSection(
+ title="All",
+ pages=[
+ ConceptPage(title=f"Page {i}", scope="s", group=g)
+ for i, g in enumerate(groups)
+ ],
+ )
+ )
+ outline.number_sections()
+ return outline
+
+ def test_a_clean_outline_has_no_hard_failures(self):
+ report = validate_outline(
+ self._outline(),
+ all_files=set(FILES),
+ test_files=TESTS,
+ max_files_per_page=TINY.max_files,
+ )
+ assert report.hard_failures == []
+
+ def test_an_invented_member_is_a_hard_failure(self):
+ outline = self._outline()
+ outline.pages[0].group.members.append("src/ghost/nowhere.py")
+ report = validate_outline(
+ outline,
+ all_files=set(FILES),
+ test_files=TESTS,
+ max_files_per_page=TINY.max_files,
+ )
+ assert report.invented_paths == ["src/ghost/nowhere.py"]
+ assert not report.ok
+
+ def test_an_unclaimed_file_is_a_hard_failure(self):
+ report = validate_outline(
+ self._outline(),
+ all_files=set(FILES) | {"src/orphan.py"},
+ test_files=TESTS,
+ max_files_per_page=TINY.max_files,
+ )
+ assert report.unclaimed_files == ["src/orphan.py"]
+ assert not report.ok
+
+ def test_two_pages_on_one_target_path_is_a_hard_failure(self):
+ outline = self._outline()
+ outline.pages[1].group.target_path = outline.pages[0].group.target_path
+ report = validate_outline(
+ outline,
+ all_files=set(FILES),
+ test_files=TESTS,
+ max_files_per_page=TINY.max_files,
+ )
+ assert report.duplicate_target_paths
+ assert not report.ok
+
+ def test_a_test_file_in_the_tree_is_a_hard_failure(self):
+ outline = self._outline()
+ outline.pages[0].group.members.append("tests/test_reader.py")
+ report = validate_outline(
+ outline,
+ all_files=set(FILES) | TESTS,
+ test_files=TESTS,
+ max_files_per_page=TINY.max_files,
+ )
+ assert report.test_paths_included == ["tests/test_reader.py"]
+ assert not report.ok
+
+ def test_a_long_title_is_reported_but_does_not_block(self):
+ outline = self._outline()
+ outline.pages[0].title = "A Very Long Title That Runs On And On Forever Indeed"
+ report = validate_outline(
+ outline,
+ all_files=set(FILES),
+ test_files=TESTS,
+ max_files_per_page=TINY.max_files,
+ )
+ assert report.bad_length_titles
+ assert report.ok, "title length is taste, not correctness"
+
+
+class TestPayload:
+ def test_group_ids_carry_no_path_information(self):
+ """A fabricated id must be recognisable as fabricated."""
+ groups = group_files(FILES, params=TINY)
+ _payload, index = build_payload(groups)
+ for gid in index:
+ assert gid.startswith("g") and gid[1:].isdigit()
+
+ def test_filenames_are_sampled_across_every_directory(self):
+ """A merged group must not be described by only one of its halves."""
+ from repowise.core.generation.concept_tree.grouping import ConceptGroup
+
+ group = ConceptGroup(
+ members=[f"src/aaa/a{i}.py" for i in range(6)]
+ + [f"src/zzz/z{i}.py" for i in range(6)],
+ dirs=["src/aaa", "src/zzz"],
+ target_path="src/aaa",
+ )
+ payload, _ = build_payload([group], max_filenames=4)
+ names = payload["groups"][0]["names"]
+ assert any(n.startswith("a") for n in names)
+ assert any(n.startswith("z") for n in names), names
+
+ def test_sibling_directories_are_shown_relative_to_what_they_share(self):
+ from repowise.core.generation.concept_tree.grouping import ConceptGroup
+
+ group = ConceptGroup(
+ members=["src/git_indexer/a.py", "src/graph/b.py"],
+ dirs=["src/git_indexer", "src/graph"],
+ target_path="src/git_indexer",
+ )
+ payload, _ = build_payload([group])
+ assert sorted(payload["groups"][0]["subdirs"]) == ["git_indexer", "graph"]
diff --git a/tests/unit/persistence/test_page_tree_wiring.py b/tests/unit/persistence/test_page_tree_wiring.py
index 4dfe29d4b..2317def52 100644
--- a/tests/unit/persistence/test_page_tree_wiring.py
+++ b/tests/unit/persistence/test_page_tree_wiring.py
@@ -46,3 +46,29 @@ def test_persist_path_rebuilds_the_tree(module_name: str, description: str):
def test_the_writer_list_is_not_silently_empty():
"""Guards the parametrisation itself."""
assert len(_PERSIST_PATHS) >= 6
+
+
+def test_the_scoped_path_sweeps_superseded_rows():
+ """Backlog B19, wired the same way and asserted the same way.
+
+ A page whose members change is rewritten under a new id, and until this
+ call existed the old row stayed behind as a duplicate on every
+ ``repowise update``. That the sweep is *correct* is tested elsewhere; this
+ asserts it is reached, because a function that is right and never called
+ looks exactly like a function that was not needed.
+ """
+ module = __import__("repowise.core.pipeline.scoped_generation", fromlist=["*"])
+ source = inspect.getsource(module)
+ assert "sweep_superseded_generated_pages(\n" in source, (
+ "a scoped regeneration writes pages without retiring the rows they "
+ "supersede, so a membership change strands the old page as a duplicate"
+ )
+
+
+def test_the_scoped_sweep_runs_before_the_tree_is_rebuilt():
+ """Order matters: a retired row must not be handed a place in the tree."""
+ module = __import__("repowise.core.pipeline.scoped_generation", fromlist=["*"])
+ source = inspect.getsource(module)
+ sweep = source.index("sweep_superseded_generated_pages(\n")
+ rebuild = source.index("await rebuild_page_tree(")
+ assert sweep < rebuild
diff --git a/tests/unit/persistence/test_superseded_sweep.py b/tests/unit/persistence/test_superseded_sweep.py
new file mode 100644
index 000000000..2b92e3c06
--- /dev/null
+++ b/tests/unit/persistence/test_superseded_sweep.py
@@ -0,0 +1,192 @@
+"""The update-path sweep (backlog B19).
+
+A structurally-keyed page's id is its ``target_path``, and a page defined by the
+files it covers can legitimately change which directory names it when its
+membership changes. On a full index the existing sweep retires the old row. On
+``repowise update`` nothing did, because the full sweep deletes every row of a
+page type the run did not reproduce — correct for a full index, and on a scoped
+run that regenerates one page out of fifty it would delete the other forty-nine.
+
+So the rule here is different and the tests that matter are the ones proving it
+is *narrow*: a page this run did not touch must survive, and so must a page
+whose generation failed.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy import select
+
+from repowise.core.persistence.models import Page, Repository
+from repowise.core.pipeline.persist import sweep_superseded_generated_pages
+
+
+async def session(async_session):
+ return async_session
+
+
+async def _new_repo(session, name: str) -> str:
+ repo_id = uuid.uuid4().hex[:32]
+ session.add(
+ Repository(id=repo_id, name=name, local_path=f"/tmp/{name}", url="")
+ )
+ await session.flush()
+ return repo_id
+
+
+async def repo_id(session):
+ return await _new_repo(session, "primary")
+
+
+async def other_repo_id(session):
+ return await _new_repo(session, "other")
+
+
+_NOW = datetime.now(UTC)
+
+
+class _Generated:
+ def __init__(self, page_id: str, page_type: str, members: list[str]):
+ self.page_id = page_id
+ self.page_type = page_type
+ self.metadata = {"file_paths": list(members)}
+
+
+async def _add(session, repo_id: str, page_id: str, page_type: str, members):
+ session.add(
+ Page(
+ id=page_id,
+ repository_id=repo_id,
+ page_type=page_type,
+ target_path=page_id.split(":", 1)[1],
+ title=page_id,
+ content="body",
+ source_hash="h" * 64,
+ model_name="m",
+ provider_name="template",
+ metadata_json=json.dumps({"file_paths": list(members)}),
+ created_at=_NOW,
+ updated_at=_NOW,
+ )
+ )
+ await session.flush()
+
+
+async def _ids(session, repo_id: str) -> set[str]:
+ rows = await session.execute(select(Page.id).where(Page.repository_id == repo_id))
+ return set(rows.scalars())
+
+
+async def test_a_row_whose_coverage_moved_to_a_new_page_is_retired(
+ session, repo_id
+):
+ """The bug: the same page under a new id, with the old row left behind."""
+ members = ["src/a.py", "src/b.py"]
+ await _add(session, repo_id, "module_page:src/old", "module_page", members)
+ swept = await sweep_superseded_generated_pages(
+ session,
+ repo_id,
+ [_Generated("module_page:src/new", "module_page", members)],
+ )
+ assert swept == ["module_page:src/old"]
+ assert "module_page:src/old" not in await _ids(session, repo_id)
+
+
+async def test_an_untouched_page_survives(session, repo_id):
+ """The reason the full sweep cannot be reused here."""
+ await _add(session, repo_id, "module_page:src/keep", "module_page", ["src/z.py"])
+ await _add(session, repo_id, "module_page:src/old", "module_page", ["src/a.py"])
+ swept = await sweep_superseded_generated_pages(
+ session,
+ repo_id,
+ [_Generated("module_page:src/new", "module_page", ["src/a.py"])],
+ )
+ assert swept == ["module_page:src/old"]
+ assert "module_page:src/keep" in await _ids(session, repo_id)
+
+
+async def test_a_page_whose_generation_failed_is_not_deleted(session, repo_id):
+ """The failure mode worse than the bug being fixed.
+
+ A page the run meant to regenerate but could not still covers files no
+ produced page claims, so it is left alone. Deleting a page because its
+ generation errored would lose content rather than a duplicate.
+ """
+ await _add(session, repo_id, "module_page:src/failed", "module_page", ["src/f.py"])
+ swept = await sweep_superseded_generated_pages(
+ session,
+ repo_id,
+ [_Generated("module_page:src/ok", "module_page", ["src/a.py"])],
+ )
+ assert swept == []
+ assert "module_page:src/failed" in await _ids(session, repo_id)
+
+
+async def test_partial_overlap_does_not_retire_a_row(session, repo_id):
+ """Supersession means *all* of it, not some of it."""
+ await _add(
+ session, repo_id, "module_page:src/old", "module_page", ["src/a.py", "src/b.py"]
+ )
+ swept = await sweep_superseded_generated_pages(
+ session,
+ repo_id,
+ [_Generated("module_page:src/new", "module_page", ["src/a.py"])],
+ )
+ assert swept == []
+
+
+async def test_a_row_with_no_recorded_membership_is_left_alone(session, repo_id):
+ """Pre-membership rows (backlog B20) cannot be proven superseded."""
+ await _add(session, repo_id, "module_page:src/old", "module_page", [])
+ swept = await sweep_superseded_generated_pages(
+ session,
+ repo_id,
+ [_Generated("module_page:src/new", "module_page", ["src/a.py"])],
+ )
+ assert swept == []
+ assert "module_page:src/old" in await _ids(session, repo_id)
+
+
+async def test_only_structurally_keyed_types_are_swept(session, repo_id):
+ """A file page's id is its path; it has no identity that can move."""
+ await _add(session, repo_id, "file_page:src/a.py", "file_page", ["src/a.py"])
+ swept = await sweep_superseded_generated_pages(
+ session,
+ repo_id,
+ [_Generated("file_page:src/b.py", "file_page", ["src/a.py"])],
+ )
+ assert swept == []
+ assert "file_page:src/a.py" in await _ids(session, repo_id)
+
+
+async def test_a_run_that_produced_nothing_retires_nothing(session, repo_id):
+ await _add(session, repo_id, "module_page:src/old", "module_page", ["src/a.py"])
+ assert await sweep_superseded_generated_pages(session, repo_id, []) == []
+ assert await sweep_superseded_generated_pages(session, repo_id, None) == []
+ assert "module_page:src/old" in await _ids(session, repo_id)
+
+
+async def test_a_different_repository_is_untouched(session, repo_id, other_repo_id):
+ await _add(session, other_repo_id, "module_page:src/old", "module_page", ["src/a.py"])
+ swept = await sweep_superseded_generated_pages(
+ session,
+ repo_id,
+ [_Generated("module_page:src/new", "module_page", ["src/a.py"])],
+ )
+ assert swept == []
+ assert "module_page:src/old" in await _ids(session, other_repo_id)